forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 14:46:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into feat-add-blobpool-blob-count-limit

Browse source
This commit is contained in:
Péter Garamvölgyi 2025-05-19 10:22:29 +02:00
commit 1ff38dcf29
No known key found for this signature in database
GPG key ID: 7FC8D069F7199070
70 changed files with 4482 additions and 824 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
cmd/devp2p/internal/ethtest/conn.go
View file

@ -66,10 +66,9 @@ func (s *Suite) dialAs(key *ecdsa.PrivateKey) (*Conn, error) {
return nil, err
}
conn.caps = []p2p.Cap{
{Name: "eth", Version: 67},
{Name: "eth", Version: 68},
{Name: "eth", Version: 69},
}
conn.ourHighestProtoVersion = 68
conn.ourHighestProtoVersion = 69
return &conn, nil
}
@ -156,7 +155,7 @@ func (c *Conn) ReadEth() (any, error) {
var msg any
switch int(code) {
case eth.StatusMsg:
msg = new(eth.StatusPacket)
msg = new(eth.StatusPacket69)
case eth.GetBlockHeadersMsg:
msg = new(eth.GetBlockHeadersPacket)
case eth.BlockHeadersMsg:
@ -229,9 +228,21 @@ func (c *Conn) ReadSnap() (any, error) {
}
}
// dialAndPeer creates a peer connection and runs the handshake.
func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) {
c, err := s.dial()
if err != nil {
return nil, err
}
if err = c.peer(s.chain, status); err != nil {
c.Close()
}
return c, err
}
// peer performs both the protocol handshake and the status message
// exchange with the node in order to peer with it.
func (c *Conn) peer(chain *Chain, status *eth.StatusPacket) error {
func (c *Conn) peer(chain *Chain, status *eth.StatusPacket69) error {
if err := c.handshake(); err != nil {
return fmt.Errorf("handshake failed: %v", err)
}
@ -304,7 +315,7 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
}
// statusExchange performs a `Status` message exchange with the given node.
func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket) error {
func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket69) error {
loop:
for {
code, data, err := c.Read()
@ -313,11 +324,15 @@ loop:
}
switch code {
case eth.StatusMsg + protoOffset(ethProto):
msg := new(eth.StatusPacket)
msg := new(eth.StatusPacket69)
if err := rlp.DecodeBytes(data, &msg); err != nil {
return fmt.Errorf("error decoding status packet: %w", err)
}
if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want {
if have, want := msg.LatestBlock, chain.blocks[chain.Len()-1].NumberU64(); have != want {
return fmt.Errorf("wrong head block in status, want: %d, have %d",
want, have)
}
if have, want := msg.LatestBlockHash, chain.blocks[chain.Len()-1].Hash(); have != want {
return fmt.Errorf("wrong head block in status, want: %#x (block %d) have %#x",
want, chain.blocks[chain.Len()-1].NumberU64(), have)
}
@ -348,13 +363,14 @@ loop:
}
if status == nil {
// default status message
status = &eth.StatusPacket{
status = &eth.StatusPacket69{
ProtocolVersion: uint32(c.negotiatedProtoVersion),
NetworkID: chain.config.ChainID.Uint64(),
TD: chain.TD(),
Head: chain.blocks[chain.Len()-1].Hash(),
Genesis: chain.blocks[0].Hash(),
ForkID: chain.ForkID(),
EarliestBlock: 0,
LatestBlock: chain.blocks[chain.Len()-1].NumberU64(),
LatestBlockHash: chain.blocks[chain.Len()-1].Hash(),
}
}
if err := c.Write(ethProto, eth.StatusMsg, status); err != nil {

2
cmd/devp2p/internal/ethtest/protocol.go
View file

@ -32,7 +32,7 @@ const (
// Unexported devp2p protocol lengths from p2p package.
const (
baseProtoLen = 16
ethProtoLen = 17
ethProtoLen = 18
snapProtoLen = 8
)

203
cmd/devp2p/internal/ethtest/suite.go
View file

@ -68,16 +68,19 @@ func (s *Suite) EthTests() []utesting.Test {
return []utesting.Test{
// status
{Name: "Status", Fn: s.TestStatus},
{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
{Name: "BlockRangeUpdateExpired", Fn: s.TestBlockRangeUpdateHistoryExp},
{Name: "BlockRangeUpdateFuture", Fn: s.TestBlockRangeUpdateFuture},
{Name: "BlockRangeUpdateInvalid", Fn: s.TestBlockRangeUpdateInvalid},
// get block headers
{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
{Name: "GetNonexistentBlockHeaders", Fn: s.TestGetNonexistentBlockHeaders},
{Name: "SimultaneousRequests", Fn: s.TestSimultaneousRequests},
{Name: "SameRequestID", Fn: s.TestSameRequestID},
{Name: "ZeroRequestID", Fn: s.TestZeroRequestID},
// get block bodies
// get history
{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
// // malicious handshakes + status
{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
{Name: "GetReceipts", Fn: s.TestGetReceipts},
// test transactions
{Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true},
{Name: "Transaction", Fn: s.TestTransaction},
@ -101,15 +104,11 @@ func (s *Suite) SnapTests() []utesting.Test {
func (s *Suite) TestStatus(t *utesting.T) {
t.Log(`This test is just a sanity check. It performs an eth protocol handshake.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
t.Fatal("peering failed:", err)
}
conn.Close()
}
// headersMatch returns whether the received headers match the given request
@ -119,15 +118,12 @@ func headersMatch(expected []*types.Header, headers []*types.Header) bool {
func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
t.Log(`This test requests block headers from the node.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err = conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Send headers request.
req := &eth.GetBlockHeadersPacket{
RequestId: 33,
@ -162,16 +158,11 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
func (s *Suite) TestGetNonexistentBlockHeaders(t *utesting.T) {
t.Log(`This test sends GetBlockHeaders requests for nonexistent blocks (using max uint64 value)
to check if the node disconnects after receiving multiple invalid requests.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create request with max uint64 value for a nonexistent block
badReq := &eth.GetBlockHeadersPacket{
@ -204,15 +195,11 @@ to check if the node disconnects after receiving multiple invalid requests.`)
func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
t.Log(`This test requests blocks headers from the node, performing two requests
concurrently, with different request IDs.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create two different requests.
req1 := &eth.GetBlockHeadersPacket{
@ -278,15 +265,11 @@ concurrently, with different request IDs.`)
func (s *Suite) TestSameRequestID(t *utesting.T) {
t.Log(`This test requests block headers, performing two concurrent requests with the
same request ID. The node should handle the request by responding to both requests.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create two different requests with the same ID.
reqID := uint64(1234)
@ -349,15 +332,12 @@ same request ID. The node should handle the request by responding to both reques
func (s *Suite) TestZeroRequestID(t *utesting.T) {
t.Log(`This test sends a GetBlockHeaders message with a request-id of zero,
and expects a response.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
req := &eth.GetBlockHeadersPacket{
GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
Origin: eth.HashOrNumber{Number: 0},
@ -384,15 +364,12 @@ and expects a response.`)
func (s *Suite) TestGetBlockBodies(t *utesting.T) {
t.Log(`This test sends GetBlockBodies requests to the node for known blocks in the test chain.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create block bodies request.
req := &eth.GetBlockBodiesPacket{
RequestId: 55,
@ -418,6 +395,47 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
}
}
func (s *Suite) TestGetReceipts(t *utesting.T) {
t.Log(`This test sends GetReceipts requests to the node for known blocks in the test chain.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Find some blocks containing receipts.
var hashes = make([]common.Hash, 0, 3)
for i := range s.chain.Len() {
block := s.chain.GetBlock(i)
if len(block.Transactions()) > 0 {
hashes = append(hashes, block.Hash())
}
if len(hashes) == cap(hashes) {
break
}
}
// Create block bodies request.
req := &eth.GetReceiptsPacket{
RequestId: 66,
GetReceiptsRequest: (eth.GetReceiptsRequest)(hashes),
}
if err := conn.Write(ethProto, eth.GetReceiptsMsg, req); err != nil {
t.Fatalf("could not write to connection: %v", err)
}
// Wait for response.
resp := new(eth.ReceiptsPacket[*eth.ReceiptList69])
if err := conn.ReadMsg(ethProto, eth.ReceiptsMsg, &resp); err != nil {
t.Fatalf("error reading block bodies msg: %v", err)
}
if got, want := resp.RequestId, req.RequestId; got != want {
t.Fatalf("unexpected request id in respond", got, want)
}
if len(resp.List) != len(req.GetReceiptsRequest) {
t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetReceiptsRequest), len(resp.List))
}
}
// randBuf makes a random buffer size kilobytes large.
func randBuf(size int) []byte {
buf := make([]byte, size*1024)
@ -500,6 +518,97 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
}
}
func (s *Suite) TestBlockRangeUpdateInvalid(t *utesting.T) {
t.Log(`This test sends an invalid BlockRangeUpdate message to the node and expects to be disconnected.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: 10,
LatestBlock: 8,
LatestBlockHash: s.chain.GetBlock(8).Hash(),
})
if code, _, err := conn.Read(); err != nil {
t.Fatalf("expected disconnect, got err: %v", err)
} else if code != discMsg {
t.Fatalf("expected disconnect message, got msg code %d", code)
}
}
func (s *Suite) TestBlockRangeUpdateFuture(t *utesting.T) {
t.Log(`This test sends a BlockRangeUpdate that is beyond the chain head.
The node should accept the update and should not disonnect.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
head := s.chain.Head().NumberU64()
var hash common.Hash
rand.Read(hash[:])
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: head + 10,
LatestBlock: head + 50,
LatestBlockHash: hash,
})
// Ensure the node does not disconnect us.
// Just send a few ping messages.
for range 10 {
time.Sleep(100 * time.Millisecond)
if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
t.Fatal("write error:", err)
}
code, _, err := conn.Read()
switch {
case err != nil:
t.Fatal("read error:", err)
case code == discMsg:
t.Fatal("got disconnect")
case code == pongMsg:
}
}
}
func (s *Suite) TestBlockRangeUpdateHistoryExp(t *utesting.T) {
t.Log(`This test sends a BlockRangeUpdate announcing incomplete (expired) history.
The node should accept the update and should not disonnect.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
head := s.chain.Head()
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: head.NumberU64() - 10,
LatestBlock: head.NumberU64(),
LatestBlockHash: head.Hash(),
})
// Ensure the node does not disconnect us.
// Just send a few ping messages.
for range 10 {
time.Sleep(100 * time.Millisecond)
if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
t.Fatal("write error:", err)
}
code, _, err := conn.Read()
switch {
case err != nil:
t.Fatal("read error:", err)
case code == discMsg:
t.Fatal("got disconnect")
case code == pongMsg:
}
}
}
func (s *Suite) TestTransaction(t *utesting.T) {
t.Log(`This test sends a valid transaction to the node and checks if the
transaction gets propagated.`)

6
cmd/geth/main.go
View file

@ -91,13 +91,7 @@ var (
utils.LogNoHistoryFlag,
utils.LogExportCheckpointsFlag,
utils.StateHistoryFlag,
utils.LightServeFlag, // deprecated
utils.LightIngressFlag, // deprecated
utils.LightEgressFlag, // deprecated
utils.LightMaxPeersFlag, // deprecated
utils.LightNoPruneFlag, // deprecated
utils.LightKDFFlag,
utils.LightNoSyncServeFlag, // deprecated
utils.EthRequiredBlocksFlag,
utils.LegacyWhitelistFlag, // deprecated
utils.CacheFlag,

3
cmd/utils/cmd.go
View file

@ -309,7 +309,8 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
if err != nil {
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
}
if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil {
encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts})
if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, 2^64-1); err != nil {
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
}
imported += 1

23
cmd/utils/flags.go
View file

@ -1245,28 +1245,6 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
}
}
// setLes shows the deprecation warnings for LES flags.
func setLes(ctx *cli.Context, cfg *ethconfig.Config) {
if ctx.IsSet(LightServeFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightServeFlag.Name)
}
if ctx.IsSet(LightIngressFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightIngressFlag.Name)
}
if ctx.IsSet(LightEgressFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightEgressFlag.Name)
}
if ctx.IsSet(LightMaxPeersFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightMaxPeersFlag.Name)
}
if ctx.IsSet(LightNoPruneFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoPruneFlag.Name)
}
if ctx.IsSet(LightNoSyncServeFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoSyncServeFlag.Name)
}
}
// MakeDatabaseHandles raises out the number of allowed file handles per process
// for Geth and returns half of the allowance to assign to the database.
func MakeDatabaseHandles(max int) int {
@ -1582,7 +1560,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
setBlobPool(ctx, &cfg.BlobPool)
setMiner(ctx, &cfg.Miner)
setRequiredBlocks(ctx, cfg)
setLes(ctx, cfg)
// Cap the cache allowance and tune the garbage collector
mem, err := gopsutil.VirtualMemory()

37
cmd/utils/flags_legacy.go
View file

@ -39,12 +39,6 @@ var DeprecatedFlags = []cli.Flag{
CacheTrieRejournalFlag,
LegacyDiscoveryV5Flag,
TxLookupLimitFlag,
LightServeFlag,
LightIngressFlag,
LightEgressFlag,
LightMaxPeersFlag,
LightNoPruneFlag,
LightNoSyncServeFlag,
LogBacktraceAtFlag,
LogDebugFlag,
MinerNewPayloadTimeoutFlag,
@ -88,37 +82,6 @@ var (
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.DeprecatedCategory,
}
// Light server and client settings, Deprecated November 2023
LightServeFlag = &cli.IntFlag{
Name: "light.serve",
Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)",
Category: flags.DeprecatedCategory,
}
LightIngressFlag = &cli.IntFlag{
Name: "light.ingress",
Usage: "Incoming bandwidth limit for serving light clients (deprecated)",
Category: flags.DeprecatedCategory,
}
LightEgressFlag = &cli.IntFlag{
Name: "light.egress",
Usage: "Outgoing bandwidth limit for serving light clients (deprecated)",
Category: flags.DeprecatedCategory,
}
LightMaxPeersFlag = &cli.IntFlag{
Name: "light.maxpeers",
Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)",
Category: flags.DeprecatedCategory,
}
LightNoPruneFlag = &cli.BoolFlag{
Name: "light.nopruning",
Usage: "Disable ancient light chain data pruning (deprecated)",
Category: flags.DeprecatedCategory,
}
LightNoSyncServeFlag = &cli.BoolFlag{
Name: "light.nosyncserve",
Usage: "Enables serving light clients before syncing (deprecated)",
Category: flags.DeprecatedCategory,
}
// Deprecated November 2023
LogBacktraceAtFlag = &cli.StringFlag{
Name: "log.backtrace",

73
core/blockchain.go
View file

@ -184,7 +184,12 @@ func (c *CacheConfig) triedbConfig(isVerkle bool) *triedb.Config {
if c.StateScheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{
StateHistory: c.StateHistory,
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
TrieCleanSize: c.TrieCleanLimit * 1024 * 1024,
StateCleanSize: c.SnapshotLimit * 1024 * 1024,
// TODO(rjl493456442): The write buffer represents the memory limit used
// for flushing both trie data and state data to disk. The config name
// should be updated to eliminate the confusion.
WriteBufferSize: c.TrieDirtyLimit * 1024 * 1024,
}
}
@ -380,11 +385,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// Do nothing here until the state syncer picks it up.
log.Info("Genesis state is missing, wait state sync")
} else {
// Head state is missing, before the state recovery, find out the
// disk layer point of snapshot(if it's enabled). Make sure the
// rewound point is lower than disk layer.
// Head state is missing, before the state recovery, find out the disk
// layer point of snapshot(if it's enabled). Make sure the rewound point
// is lower than disk layer.
//
// Note it's unnecessary in path mode which always keep trie data and
// state data consistent.
var diskRoot common.Hash
if bc.cacheConfig.SnapshotLimit > 0 {
if bc.cacheConfig.SnapshotLimit > 0 && bc.cacheConfig.StateScheme == rawdb.HashScheme {
diskRoot = rawdb.ReadSnapshotRoot(bc.db)
}
if diskRoot != (common.Hash{}) {
@ -457,7 +465,32 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
bc.logger.OnGenesisBlock(bc.genesisBlock, alloc)
}
}
bc.setupSnapshot()
// Rewind the chain in case of an incompatible config upgrade.
if compatErr != nil {
log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
if compatErr.RewindToTime > 0 {
bc.SetHeadWithTimestamp(compatErr.RewindToTime)
} else {
bc.SetHead(compatErr.RewindToBlock)
}
rawdb.WriteChainConfig(db, genesisHash, chainConfig)
}
// Start tx indexer if it's enabled.
if txLookupLimit != nil {
bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
}
return bc, nil
}
func (bc *BlockChain) setupSnapshot() {
// Short circuit if the chain is established with path scheme, as the
// state snapshot has been integrated into path database natively.
if bc.cacheConfig.StateScheme == rawdb.PathScheme {
return
}
// Load any existing snapshot, regenerating it if loading failed
if bc.cacheConfig.SnapshotLimit > 0 {
// If the chain was rewound past the snapshot persistent layer (causing
@ -465,7 +498,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// in recovery mode and in that case, don't invalidate the snapshot on a
// head mismatch.
var recover bool
head := bc.CurrentBlock()
if layer := rawdb.ReadSnapshotRecoveryNumber(bc.db); layer != nil && *layer >= head.Number.Uint64() {
log.Warn("Enabling snapshot recovery", "chainhead", head.Number, "diskbase", *layer)
@ -482,22 +514,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// Re-initialize the state database with snapshot
bc.statedb = state.NewDatabase(bc.triedb, bc.snaps)
}
// Rewind the chain in case of an incompatible config upgrade.
if compatErr != nil {
log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
if compatErr.RewindToTime > 0 {
bc.SetHeadWithTimestamp(compatErr.RewindToTime)
} else {
bc.SetHead(compatErr.RewindToBlock)
}
rawdb.WriteChainConfig(db, genesisHash, chainConfig)
}
// Start tx indexer if it's enabled.
if txLookupLimit != nil {
bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
}
return bc, nil
}
// empty returns an indicator whether the blockchain is empty.
@ -1293,12 +1309,11 @@ const (
//
// The optional ancientLimit can also be specified and chain segment before that
// will be directly stored in the ancient, getting rid of the chain migration.
func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) {
func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []rlp.RawValue, ancientLimit uint64) (int, error) {
// Verify the supplied headers before insertion without lock
var headers []*types.Header
for _, block := range blockChain {
headers = append(headers, block.Header())
// Here we also validate that blob transactions in the block do not
// contain a sidecar. While the sidecar does not affect the block hash
// or tx hash, sending blobs within a block is not allowed.
@ -1341,11 +1356,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
//
// this function only accepts canonical chain data. All side chain will be reverted
// eventually.
writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
writeAncient := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
// Ensure genesis is in the ancient store
if blockChain[0].NumberU64() == 1 {
if frozen, _ := bc.db.Ancients(); frozen == 0 {
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err
@ -1388,7 +1403,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// existing local chain segments (reorg around the chain tip). The reorganized part
// will be included in the provided chain segment, and stale canonical markers will be
// silently rewritten. Therefore, no explicit reorg logic is needed.
writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
writeLive := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
var (
skipPresenceCheck = false
batch = bc.db.NewBatch()
@ -1413,7 +1428,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// Write all the data out into the database
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
rawdb.WriteBlock(batch, block)
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
rawdb.WriteRawReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
// Write everything belongs to the blocks into the database. So that
// we can ensure all components of body is completed(body, receipts)
@ -2634,7 +2649,7 @@ func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, e
first = headers[0].Number.Uint64()
)
if first == 1 && frozen == 0 {
_, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
_, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err

14
core/blockchain_reader.go
View file

@ -234,12 +234,22 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
return receipts
}
func (bc *BlockChain) GetRawReceiptsByHash(hash common.Hash) types.Receipts {
// GetRawReceipts retrieves the receipts for all transactions in a given block
// without deriving the internal fields and the Bloom.
func (bc *BlockChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
if receipts, ok := bc.receiptsCache.Get(hash); ok {
return receipts
}
return rawdb.ReadRawReceipts(bc.db, hash, number)
}
// GetReceiptsRLP retrieves the receipts of a block.
func (bc *BlockChain) GetReceiptsRLP(hash common.Hash) rlp.RawValue {
number := rawdb.ReadHeaderNumber(bc.db, hash)
if number == nil {
return nil
}
return rawdb.ReadRawReceipts(bc.db, hash, *number)
return rawdb.ReadReceiptsRLP(bc.db, hash, *number)
}
// GetUnclesInChain retrieves all the uncles from a given block backwards until

20
core/blockchain_repair_test.go
View file

@ -1791,7 +1791,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
}
)
defer engine.Close()
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
config.SnapshotLimit = 256
config.SnapshotWait = true
}
@ -1820,7 +1820,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
t.Fatalf("Failed to flush trie state: %v", err)
}
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
@ -1952,9 +1952,11 @@ func testIssue23496(t *testing.T, scheme string) {
if _, err := chain.InsertChain(blocks[1:2]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
if scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(blocks[1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
}
// Insert block B3 and commit the state into disk
if _, err := chain.InsertChain(blocks[2:3]); err != nil {
@ -1997,16 +1999,24 @@ func testIssue23496(t *testing.T, scheme string) {
}
expHead := uint64(1)
if scheme == rawdb.PathScheme {
expHead = uint64(2)
// The pathdb database makes sure that snapshot and trie are consistent,
// so only the last block is reverted in case of a crash.
expHead = uint64(3)
}
if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
}
if scheme == rawdb.PathScheme {
// Reinsert B4
if _, err := chain.InsertChain(blocks[3:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
} else {
// Reinsert B2-B4
if _, err := chain.InsertChain(blocks[1:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
}
if head := chain.CurrentHeader(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head header mismatch: have %d, want %d", head.Number, 4)
}
@ -2016,7 +2026,9 @@ func testIssue23496(t *testing.T, scheme string) {
if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(4))
}
if scheme == rawdb.HashScheme {
if layer := chain.Snapshots().Snapshot(blocks[2].Root()); layer == nil {
t.Error("Failed to regenerate the snapshot of known state")
}
}
}

2
core/blockchain_sethead_test.go
View file

@ -2023,7 +2023,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
}
if tt.commitBlock > 0 {
chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}

16
core/blockchain_snapshot_test.go
View file

@ -105,7 +105,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
if basic.commitBlock > 0 && basic.commitBlock == point {
chain.TrieDB().Commit(blocks[point-1].Root(), false)
}
if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
if basic.snapshotBlock > 0 && basic.snapshotBlock == point && basic.scheme == rawdb.HashScheme {
// Flushing the entire snap tree into the disk, the
// relevant (a) snapshot root and (b) snapshot generator
// will be persisted atomically.
@ -149,15 +149,19 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [
block := chain.GetBlockByNumber(basic.expSnapshotBottom)
if block == nil {
t.Errorf("The corresponding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom)
} else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
} else if basic.scheme == rawdb.HashScheme {
if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
}
}
// Check the snapshot, ensure it's integrated
if basic.scheme == rawdb.HashScheme {
if err := chain.snaps.Verify(block.Root()); err != nil {
t.Errorf("The disk layer is not integrated %v", err)
}
}
}
//nolint:unused
func (basic *snapshotTestBasic) dump() string {
@ -565,12 +569,14 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
// Expected head block : G (Hash mode), C6 (Hash mode)
// Expected snapshot disk : C4 (Hash mode)
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
expHead := uint64(0)
if scheme == rawdb.PathScheme {
expHead = uint64(4)
// The pathdb database makes sure that snapshot and trie are consistent,
// so only the last two blocks are reverted in case of a crash.
expHead = uint64(6)
}
test := &crashSnapshotTest{
snapshotTestBasic{

22
core/blockchain_test.go
View file

@ -734,7 +734,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// Freezer style fast import the chain.
@ -747,7 +747,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(len(blocks)/2)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -871,7 +871,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
assert(t, "fast", fast, height, height, 0)
@ -884,7 +884,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
assert(t, "ancient", ancient, height, height, 0)
@ -1696,7 +1696,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
defer ancientDb.Close()
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
rawdb.WriteLastPivotNumber(ancientDb, blocks[len(blocks)-1].NumberU64()) // Force fast sync behavior
@ -1991,7 +1991,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
return err
}
asserter = func(t *testing.T, block *types.Block) {
@ -2157,7 +2157,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
return err
}
asserter = func(t *testing.T, block *types.Block) {
@ -4205,10 +4205,10 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
if n, err := chain.InsertReceiptChain(blocks, receipts, ancientLimit); err != nil {
if n, err := chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(chainA, receiptsA, ancientLimit); err != nil {
if n, err := chain.InsertReceiptChain(chainA, types.EncodeBlockReceiptLists(receiptsA), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// If the common ancestor is below the ancient limit, rewind the chain head.
@ -4218,7 +4218,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
rawdb.WriteLastPivotNumber(db, ancestor)
chain.SetHead(ancestor)
}
if n, err := chain.InsertReceiptChain(chainB, receiptsB, ancientLimit); err != nil {
if n, err := chain.InsertReceiptChain(chainB, types.EncodeBlockReceiptLists(receiptsB), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
head := chain.CurrentSnapBlock()
@ -4336,7 +4336,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
if n, err := chain.InsertHeadersBeforeCutoff(headersBefore); err != nil {
t.Fatalf("failed to insert headers before cutoff %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(blocksAfter, receiptsAfter, ancientLimit); err != nil {
if n, err := chain.InsertReceiptChain(blocksAfter, types.EncodeBlockReceiptLists(receiptsAfter), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
headSnap := chain.CurrentSnapBlock()

4
core/filtermaps/chain_view.go
View file

@ -29,7 +29,7 @@ type blockchain interface {
GetHeader(hash common.Hash, number uint64) *types.Header
GetCanonicalHash(number uint64) common.Hash
GetReceiptsByHash(hash common.Hash) types.Receipts
GetRawReceiptsByHash(hash common.Hash) types.Receipts
GetRawReceipts(hash common.Hash, number uint64) types.Receipts
}
// ChainView represents an immutable view of a chain with a block id and a set
@ -117,7 +117,7 @@ func (cv *ChainView) RawReceipts(number uint64) types.Receipts {
log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
return nil
}
return cv.chain.GetRawReceiptsByHash(blockHash)
return cv.chain.GetRawReceipts(blockHash, number)
}
// SharedRange returns the block range shared by two chain views.

2
core/filtermaps/filtermaps.go
View file

@ -227,7 +227,7 @@ type Config struct {
// NewFilterMaps creates a new FilterMaps and starts the indexer.
func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) *FilterMaps {
rs, initialized, err := rawdb.ReadFilterMapsRange(db)
if err != nil || rs.Version != databaseVersion {
if err != nil || (initialized && rs.Version != databaseVersion) {
rs, initialized = rawdb.FilterMapsRange{}, false
log.Warn("Invalid log index database version; resetting log index")
}

2
core/filtermaps/indexer_test.go
View file

@ -515,7 +515,7 @@ func (tc *testChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
return tc.receipts[hash]
}
func (tc *testChain) GetRawReceiptsByHash(hash common.Hash) types.Receipts {
func (tc *testChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
tc.lock.RLock()
defer tc.lock.RUnlock()

25
core/rawdb/accessors_chain.go
View file

@ -545,8 +545,8 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa
}
// ReadRawReceipts retrieves all the transaction receipts belonging to a block.
// The receipt metadata fields are not guaranteed to be populated, so they
// should not be used. Use ReadReceipts instead if the metadata is needed.
// The receipt metadata fields and the Bloom are not guaranteed to be populated,
// so they should not be used. Use ReadReceipts instead if the metadata is needed.
func ReadRawReceipts(db ethdb.Reader, hash common.Hash, number uint64) types.Receipts {
// Retrieve the flattened receipt slice
data := ReadReceiptsRLP(db, hash, number)
@ -621,6 +621,14 @@ func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rec
}
}
// WriteRawReceipts stores all the transaction receipts belonging to a block.
func WriteRawReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, receipts rlp.RawValue) {
// Store the flattened receipt slice
if err := db.Put(blockReceiptsKey(number, hash), receipts); err != nil {
log.Crit("Failed to store block receipts", "err", err)
}
}
// DeleteReceipts removes all receipt data associated with a block hash.
func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
if err := db.Delete(blockReceiptsKey(number, hash)); err != nil {
@ -701,18 +709,11 @@ func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) {
}
// WriteAncientBlocks writes entire block data into ancient store and returns the total written size.
func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []types.Receipts) (int64, error) {
var stReceipts []*types.ReceiptForStorage
func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []rlp.RawValue) (int64, error) {
return db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
for i, block := range blocks {
// Convert receipts to storage format and sum up total difficulty.
stReceipts = stReceipts[:0]
for _, receipt := range receipts[i] {
stReceipts = append(stReceipts, (*types.ReceiptForStorage)(receipt))
}
header := block.Header()
if err := writeAncientBlock(op, block, header, stReceipts); err != nil {
if err := writeAncientBlock(op, block, header, receipts[i]); err != nil {
return err
}
}
@ -720,7 +721,7 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
})
}
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage) error {
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts rlp.RawValue) error {
num := block.NumberU64()
if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
return fmt.Errorf("can't add block %d hash: %v", num, err)

12
core/rawdb/accessors_chain_test.go
View file

@ -377,7 +377,11 @@ func TestBlockReceiptStorage(t *testing.T) {
t.Fatalf("receipts returned when body was deleted: %v", rs)
}
// Ensure that receipts without metadata can be returned without the block body too
if err := checkReceiptsRLP(ReadRawReceipts(db, hash, 0), receipts); err != nil {
raw := ReadRawReceipts(db, hash, 0)
for _, r := range raw {
r.Bloom = types.CreateBloom(r)
}
if err := checkReceiptsRLP(raw, receipts); err != nil {
t.Fatal(err)
}
// Sanity check that body alone without the receipt is a full purge
@ -439,7 +443,7 @@ func TestAncientStorage(t *testing.T) {
}
// Write and verify the header in the database
WriteAncientBlocks(db, []*types.Block{block}, []types.Receipts{nil})
WriteAncientBlocks(db, []*types.Block{block}, types.EncodeBlockReceiptLists([]types.Receipts{nil}))
if blob := ReadHeaderRLP(db, hash, number); len(blob) == 0 {
t.Fatalf("no header returned")
@ -609,7 +613,7 @@ func BenchmarkWriteAncientBlocks(b *testing.B) {
blocks := allBlocks[i : i+length]
receipts := batchReceipts[:length]
writeSize, err := WriteAncientBlocks(db, blocks, receipts)
writeSize, err := WriteAncientBlocks(db, blocks, types.EncodeBlockReceiptLists(receipts))
if err != nil {
b.Fatal(err)
}
@ -909,7 +913,7 @@ func TestHeadersRLPStorage(t *testing.T) {
}
receipts := make([]types.Receipts, 100)
// Write first half to ancients
WriteAncientBlocks(db, chain[:50], receipts[:50])
WriteAncientBlocks(db, chain[:50], types.EncodeBlockReceiptLists(receipts[:50]))
// Write second half to db
for i := 50; i < 100; i++ {
WriteCanonicalHash(db, chain[i].Hash(), chain[i].NumberU64())

5
core/rawdb/accessors_indexes.go
View file

@ -446,7 +446,7 @@ type FilterMapsRange struct {
// database entry is not present, that is interpreted as a valid non-initialized
// state and returns a blank range structure and no error.
func ReadFilterMapsRange(db ethdb.KeyValueReader) (FilterMapsRange, bool, error) {
if has, err := db.Has(filterMapsRangeKey); !has || err != nil {
if has, err := db.Has(filterMapsRangeKey); err != nil || !has {
return FilterMapsRange{}, false, err
}
encRange, err := db.Get(filterMapsRangeKey)
@ -457,7 +457,8 @@ func ReadFilterMapsRange(db ethdb.KeyValueReader) (FilterMapsRange, bool, error)
if err := rlp.DecodeBytes(encRange, &fmRange); err != nil {
return FilterMapsRange{}, false, err
}
return fmRange, true, err
return fmRange, true, nil
}
// WriteFilterMapsRange stores the filter maps range data.

25
core/state/database.go
View file

@ -175,26 +175,27 @@ func NewDatabaseForTesting() *CachingDB {
func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
var readers []StateReader
// Set up the state snapshot reader if available. This feature
// is optional and may be partially useful if it's not fully
// generated.
if db.snap != nil {
// If standalone state snapshot is available (hash scheme),
// then construct the legacy snap reader.
// Configure the state reader using the standalone snapshot in hash mode.
// This reader offers improved performance but is optional and only
// partially useful if the snapshot is not fully generated.
if db.TrieDB().Scheme() == rawdb.HashScheme && db.snap != nil {
snap := db.snap.Snapshot(stateRoot)
if snap != nil {
readers = append(readers, newFlatReader(snap))
}
} else {
// If standalone state snapshot is not available, try to construct
// the state reader with database.
}
// Configure the state reader using the path database in path mode.
// This reader offers improved performance but is optional and only
// partially useful if the snapshot data in path database is not
// fully generated.
if db.TrieDB().Scheme() == rawdb.PathScheme {
reader, err := db.triedb.StateReader(stateRoot)
if err == nil {
readers = append(readers, newFlatReader(reader)) // state reader is optional
readers = append(readers, newFlatReader(reader))
}
}
// Set up the trie reader, which is expected to always be available
// as the gatekeeper unless the state is corrupted.
// Configure the trie reader, which is expected to be available as the
// gatekeeper unless the state is corrupted.
tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache)
if err != nil {
return nil, err

4
core/state/snapshot/generate_test.go
View file

@ -166,7 +166,9 @@ func newHelper(scheme string) *testHelper {
diskdb := rawdb.NewMemoryDatabase()
config := &triedb.Config{}
if scheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{} // disable caching
config.PathDB = &pathdb.Config{
SnapshotNoBuild: true,
} // disable caching
} else {
config.HashDB = &hashdb.Config{} // disable caching
}

3
core/state/statedb_test.go
View file

@ -979,7 +979,8 @@ func testMissingTrieNodes(t *testing.T, scheme string) {
)
if scheme == rawdb.PathScheme {
tdb = triedb.NewDatabase(memDb, &triedb.Config{PathDB: &pathdb.Config{
CleanCacheSize: 0,
TrieCleanSize: 0,
StateCleanSize: 0,
WriteBufferSize: 0,
}}) // disable caching
} else {

8
core/txindexer_test.go
View file

@ -117,7 +117,7 @@ func TestTxIndexer(t *testing.T) {
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...)))
// Index the initial blocks from ancient store
indexer := &txIndexer{
@ -236,7 +236,8 @@ func TestTxIndexerRepair(t *testing.T) {
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
encReceipts := types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), encReceipts)
// Index the initial blocks from ancient store
indexer := &txIndexer{
@ -426,7 +427,8 @@ func TestTxIndexerReport(t *testing.T) {
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
encReceipts := types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), encReceipts)
// Index the initial blocks from ancient store
indexer := &txIndexer{

21
core/types/bloom9.go
View file

@ -59,11 +59,12 @@ func (b *Bloom) SetBytes(d []byte) {
// Add adds d to the filter. Future calls of Test(d) will return true.
func (b *Bloom) Add(d []byte) {
b.add(d, make([]byte, 6))
var buf [6]byte
b.AddWithBuffer(d, &buf)
}
// add is internal version of Add, which takes a scratch buffer for reuse (needs to be at least 6 bytes)
func (b *Bloom) add(d []byte, buf []byte) {
func (b *Bloom) AddWithBuffer(d []byte, buf *[6]byte) {
i1, v1, i2, v2, i3, v3 := bloomValues(d, buf)
b[i1] |= v1
b[i2] |= v2
@ -84,7 +85,8 @@ func (b Bloom) Bytes() []byte {
// Test checks if the given topic is present in the bloom filter
func (b Bloom) Test(topic []byte) bool {
i1, v1, i2, v2, i3, v3 := bloomValues(topic, make([]byte, 6))
var buf [6]byte
i1, v1, i2, v2, i3, v3 := bloomValues(topic, &buf)
return v1 == v1&b[i1] &&
v2 == v2&b[i2] &&
v3 == v3&b[i3]
@ -104,12 +106,12 @@ func (b *Bloom) UnmarshalText(input []byte) error {
func CreateBloom(receipt *Receipt) Bloom {
var (
bin Bloom
buf = make([]byte, 6)
buf [6]byte
)
for _, log := range receipt.Logs {
bin.add(log.Address.Bytes(), buf)
bin.AddWithBuffer(log.Address.Bytes(), &buf)
for _, b := range log.Topics {
bin.add(b[:], buf)
bin.AddWithBuffer(b[:], &buf)
}
}
return bin
@ -139,21 +141,20 @@ func Bloom9(data []byte) []byte {
}
// bloomValues returns the bytes (index-value pairs) to set for the given data
func bloomValues(data []byte, hashbuf []byte) (uint, byte, uint, byte, uint, byte) {
func bloomValues(data []byte, hashbuf *[6]byte) (uint, byte, uint, byte, uint, byte) {
sha := hasherPool.Get().(crypto.KeccakState)
sha.Reset()
sha.Write(data)
sha.Read(hashbuf)
sha.Read(hashbuf[:])
hasherPool.Put(sha)
// The actual bits to flip
v1 := byte(1 << (hashbuf[1] & 0x7))
v2 := byte(1 << (hashbuf[3] & 0x7))
v3 := byte(1 << (hashbuf[5] & 0x7))
// The indices for the bytes to OR in
i1 := BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf)&0x7ff)>>3) - 1
i1 := BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[0:])&0x7ff)>>3) - 1
i2 := BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[2:])&0x7ff)>>3) - 1
i3 := BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[4:])&0x7ff)>>3) - 1
return i1, v1, i2, v2, i3, v3
}

24
core/types/receipt.go
View file

@ -27,6 +27,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
)
@ -258,7 +259,7 @@ func (r *Receipt) Size() common.StorageSize {
}
// ReceiptForStorage is a wrapper around a Receipt with RLP serialization
// that omits the Bloom field and deserialization that re-computes it.
// that omits the Bloom field. The Bloom field is recomputed by DeriveFields.
type ReceiptForStorage Receipt
// EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt
@ -291,7 +292,6 @@ func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
}
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.Logs = stored.Logs
r.Bloom = CreateBloom((*Receipt)(r))
return nil
}
@ -372,6 +372,26 @@ func (rs Receipts) DeriveFields(config *params.ChainConfig, hash common.Hash, nu
rs[i].Logs[j].Index = logIndex
logIndex++
}
// also derive the Bloom if not derived yet
rs[i].Bloom = CreateBloom(rs[i])
}
return nil
}
// EncodeBlockReceiptLists encodes a list of block receipt lists into RLP.
func EncodeBlockReceiptLists(receipts []Receipts) []rlp.RawValue {
var storageReceipts []*ReceiptForStorage
result := make([]rlp.RawValue, len(receipts))
for i, receipt := range receipts {
storageReceipts = storageReceipts[:0]
for _, r := range receipt {
storageReceipts = append(storageReceipts, (*ReceiptForStorage)(r))
}
bytes, err := rlp.EncodeToBytes(storageReceipts)
if err != nil {
log.Crit("Failed to encode block receipts", "err", err)
}
result[i] = bytes
}
return result
}

22
core/types/receipt_test.go
View file

@ -22,6 +22,7 @@ import (
"math"
"math/big"
"reflect"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
@ -154,9 +155,16 @@ var (
blockNumber = big.NewInt(1)
blockTime = uint64(2)
blockHash = common.BytesToHash([]byte{0x03, 0x14})
)
var receiptsOnce sync.Once
var testReceipts Receipts
func getTestReceipts() Receipts {
// Compute the blooms only once
receiptsOnce.Do(func() {
// Create the corresponding receipts
receipts = Receipts{
r := Receipts{
&Receipt{
Status: ReceiptStatusFailed,
CumulativeGasUsed: 1,
@ -294,7 +302,13 @@ var (
TransactionIndex: 6,
},
}
)
for _, receipt := range r {
receipt.Bloom = CreateBloom(receipt)
}
testReceipts = r
})
return testReceipts
}
func TestDecodeEmptyTypedReceipt(t *testing.T) {
input := []byte{0x80}
@ -310,6 +324,7 @@ func TestDeriveFields(t *testing.T) {
// Re-derive receipts.
basefee := big.NewInt(1000)
blobGasPrice := big.NewInt(920)
receipts := getTestReceipts()
derivedReceipts := clearComputedFieldsOnReceipts(receipts)
err := Receipts(derivedReceipts).DeriveFields(params.TestChainConfig, blockHash, blockNumber.Uint64(), blockTime, basefee, blobGasPrice, txs)
if err != nil {
@ -335,6 +350,7 @@ func TestDeriveFields(t *testing.T) {
// Test that we can marshal/unmarshal receipts to/from json without errors.
// This also confirms that our test receipts contain all the required fields.
func TestReceiptJSON(t *testing.T) {
receipts := getTestReceipts()
for i := range receipts {
b, err := receipts[i].MarshalJSON()
if err != nil {
@ -351,6 +367,7 @@ func TestReceiptJSON(t *testing.T) {
// Test we can still parse receipt without EffectiveGasPrice for backwards compatibility, even
// though it is required per the spec.
func TestEffectiveGasPriceNotRequired(t *testing.T) {
receipts := getTestReceipts()
r := *receipts[0]
r.EffectiveGasPrice = nil
b, err := r.MarshalJSON()
@ -511,6 +528,7 @@ func clearComputedFieldsOnReceipt(receipt *Receipt) *Receipt {
cpy.EffectiveGasPrice = big.NewInt(0)
cpy.BlobGasUsed = 0
cpy.BlobGasPrice = nil
cpy.Bloom = CreateBloom(&cpy)
return &cpy
}

50
eth/api_debug.go
View file

@ -271,26 +271,26 @@ func storageRangeAt(statedb *state.StateDB, root common.Hash, address common.Add
//
// With one parameter, returns the list of accounts modified in the specified block.
func (api *DebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) {
var startBlock, endBlock *types.Block
var startHeader, endHeader *types.Header
startBlock = api.eth.blockchain.GetBlockByNumber(startNum)
if startBlock == nil {
startHeader = api.eth.blockchain.GetHeaderByNumber(startNum)
if startHeader == nil {
return nil, fmt.Errorf("start block %x not found", startNum)
}
if endNum == nil {
endBlock = startBlock
startBlock = api.eth.blockchain.GetBlockByHash(startBlock.ParentHash())
if startBlock == nil {
return nil, fmt.Errorf("block %x has no parent", endBlock.Number())
endHeader = startHeader
startHeader = api.eth.blockchain.GetHeaderByHash(startHeader.ParentHash)
if startHeader == nil {
return nil, fmt.Errorf("block %x has no parent", endHeader.Number)
}
} else {
endBlock = api.eth.blockchain.GetBlockByNumber(*endNum)
if endBlock == nil {
endHeader = api.eth.blockchain.GetHeaderByNumber(*endNum)
if endHeader == nil {
return nil, fmt.Errorf("end block %d not found", *endNum)
}
}
return api.getModifiedAccounts(startBlock, endBlock)
return api.getModifiedAccounts(startHeader, endHeader)
}
// GetModifiedAccountsByHash returns all accounts that have changed between the
@ -299,38 +299,38 @@ func (api *DebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64
//
// With one parameter, returns the list of accounts modified in the specified block.
func (api *DebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) {
var startBlock, endBlock *types.Block
startBlock = api.eth.blockchain.GetBlockByHash(startHash)
if startBlock == nil {
var startHeader, endHeader *types.Header
startHeader = api.eth.blockchain.GetHeaderByHash(startHash)
if startHeader == nil {
return nil, fmt.Errorf("start block %x not found", startHash)
}
if endHash == nil {
endBlock = startBlock
startBlock = api.eth.blockchain.GetBlockByHash(startBlock.ParentHash())
if startBlock == nil {
return nil, fmt.Errorf("block %x has no parent", endBlock.Number())
endHeader = startHeader
startHeader = api.eth.blockchain.GetHeaderByHash(startHeader.ParentHash)
if startHeader == nil {
return nil, fmt.Errorf("block %x has no parent", endHeader.Number)
}
} else {
endBlock = api.eth.blockchain.GetBlockByHash(*endHash)
if endBlock == nil {
endHeader = api.eth.blockchain.GetHeaderByHash(*endHash)
if endHeader == nil {
return nil, fmt.Errorf("end block %x not found", *endHash)
}
}
return api.getModifiedAccounts(startBlock, endBlock)
return api.getModifiedAccounts(startHeader, endHeader)
}
func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) {
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
func (api *DebugAPI) getModifiedAccounts(startHeader, endHeader *types.Header) ([]common.Address, error) {
if startHeader.Number.Uint64() >= endHeader.Number.Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startHeader.Number.Uint64(), endHeader.Number.Uint64())
}
triedb := api.eth.BlockChain().TrieDB()
oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb)
oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startHeader.Root), triedb)
if err != nil {
return nil, err
}
newTrie, err := trie.NewStateTrie(trie.StateTrieID(endBlock.Root()), triedb)
newTrie, err := trie.NewStateTrie(trie.StateTrieID(endHeader.Root), triedb)
if err != nil {
return nil, err
}

112
eth/api_debug_test.go
View file

@ -18,25 +18,74 @@ package eth
import (
"bytes"
"crypto/ecdsa"
"fmt"
"math/big"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/tracing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/triedb"
"github.com/holiman/uint256"
"github.com/stretchr/testify/assert"
)
var dumper = spew.ConfigState{Indent: " "}
type Account struct {
key *ecdsa.PrivateKey
addr common.Address
}
func newAccounts(n int) (accounts []Account) {
for i := 0; i < n; i++ {
key, _ := crypto.GenerateKey()
addr := crypto.PubkeyToAddress(key.PublicKey)
accounts = append(accounts, Account{key: key, addr: addr})
}
slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) })
return accounts
}
// newTestBlockChain creates a new test blockchain. OBS: After test is done, teardown must be
// invoked in order to release associated resources.
func newTestBlockChain(t *testing.T, n int, gspec *core.Genesis, generator func(i int, b *core.BlockGen)) *core.BlockChain {
engine := ethash.NewFaker()
// Generate blocks for testing
_, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, generator)
// Import the canonical chain
cacheConfig := &core.CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0,
Preimages: true,
TrieDirtyDisabled: true, // Archive mode
}
chain, err := core.NewBlockChain(rawdb.NewMemoryDatabase(), cacheConfig, gspec, nil, engine, vm.Config{}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
return chain
}
func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.Dump {
result := statedb.RawDump(&state.DumpConfig{
SkipCode: true,
@ -224,3 +273,66 @@ func TestStorageRangeAt(t *testing.T) {
}
}
}
func TestGetModifiedAccounts(t *testing.T) {
t.Parallel()
// Initialize test accounts
accounts := newAccounts(4)
genesis := &core.Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{
accounts[0].addr: {Balance: big.NewInt(params.Ether)},
accounts[1].addr: {Balance: big.NewInt(params.Ether)},
accounts[2].addr: {Balance: big.NewInt(params.Ether)},
accounts[3].addr: {Balance: big.NewInt(params.Ether)},
},
}
genBlocks := 1
signer := types.HomesteadSigner{}
blockChain := newTestBlockChain(t, genBlocks, genesis, func(_ int, b *core.BlockGen) {
// Transfer from account[0] to account[1]
// value: 1000 wei
// fee: 0 wei
for _, account := range accounts[:3] {
tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{
Nonce: 0,
To: &accounts[3].addr,
Value: big.NewInt(1000),
Gas: params.TxGas,
GasPrice: b.BaseFee(),
Data: nil}),
signer, account.key)
b.AddTx(tx)
}
})
defer blockChain.Stop()
// Create a debug API instance.
api := NewDebugAPI(&Ethereum{blockchain: blockChain})
// Test GetModifiedAccountsByNumber
t.Run("GetModifiedAccountsByNumber", func(t *testing.T) {
addrs, err := api.GetModifiedAccountsByNumber(uint64(genBlocks), nil)
assert.NoError(t, err)
assert.Len(t, addrs, len(accounts)+1) // +1 for the coinbase
for _, account := range accounts {
if !slices.Contains(addrs, account.addr) {
t.Fatalf("account %s not found in modified accounts", account.addr.Hex())
}
}
})
// Test GetModifiedAccountsByHash
t.Run("GetModifiedAccountsByHash", func(t *testing.T) {
header := blockChain.GetHeaderByNumber(uint64(genBlocks))
addrs, err := api.GetModifiedAccountsByHash(header.Hash(), nil)
assert.NoError(t, err)
assert.Len(t, addrs, len(accounts)+1) // +1 for the coinbase
for _, account := range accounts {
if !slices.Contains(addrs, account.addr) {
t.Fatalf("account %s not found in modified accounts", account.addr.Hex())
}
}
})
}

7
eth/downloader/downloader.go
View file

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/triedb"
)
@ -202,7 +203,7 @@ type BlockChain interface {
// into the local chain. Blocks older than the specified `ancientLimit`
// are stored directly in the ancient store, while newer blocks are stored
// in the live key-value store.
InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error)
InsertReceiptChain(types.Blocks, []rlp.RawValue, uint64) (int, error)
// Snapshots returns the blockchain snapshot tree to paused it during sync.
Snapshots() *snapshot.Tree
@ -1034,7 +1035,7 @@ func (d *Downloader) commitSnapSyncData(results []*fetchResult, stateSync *state
"lastnumn", last.Number, "lasthash", last.Hash(),
)
blocks := make([]*types.Block, len(results))
receipts := make([]types.Receipts, len(results))
receipts := make([]rlp.RawValue, len(results))
for i, result := range results {
blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.body())
receipts[i] = result.Receipts
@ -1051,7 +1052,7 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error {
log.Debug("Committing snap sync pivot as new head", "number", block.Number(), "hash", block.Hash())
// Commit the pivot block as the new head, will require full sync from here on
if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{result.Receipts}, d.ancientLimit); err != nil {
if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []rlp.RawValue{result.Receipts}, d.ancientLimit); err != nil {
return err
}
if err := d.blockchain.SnapSyncCommitHead(block.Hash()); err != nil {

9
eth/downloader/downloader_test.go
View file

@ -255,23 +255,24 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *et
// peer in the download tester. The returned function can be used to retrieve
// batches of block receipts from the particularly requested peer.
func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) {
blobs := eth.ServiceGetReceiptsQuery(dlp.chain, hashes)
blobs := eth.ServiceGetReceiptsQuery68(dlp.chain, hashes)
receipts := make([][]*types.Receipt, len(blobs))
receipts := make([]types.Receipts, len(blobs))
for i, blob := range blobs {
rlp.DecodeBytes(blob, &receipts[i])
}
hasher := trie.NewStackTrie(nil)
hashes = make([]common.Hash, len(receipts))
for i, receipt := range receipts {
hashes[i] = types.DeriveSha(types.Receipts(receipt), hasher)
hashes[i] = types.DeriveSha(receipt, hasher)
}
req := &eth.Request{
Peer: dlp.id,
}
resp := eth.ReceiptsRLPResponse(types.EncodeBlockReceiptLists(receipts))
res := &eth.Response{
Req: req,
Res: (*eth.ReceiptsResponse)(&receipts),
Res: &resp,
Meta: hashes,
Time: 1,
Done: make(chan error, 1), // Ignore the returned status

2
eth/downloader/fetchers_concurrent_receipts.go
View file

@ -88,7 +88,7 @@ func (q *receiptQueue) request(peer *peerConnection, req *fetchRequest, resCh ch
// deliver is responsible for taking a generic response packet from the concurrent
// fetcher, unpacking the receipt data and delivering it to the downloader's queue.
func (q *receiptQueue) deliver(peer *peerConnection, packet *eth.Response) (int, error) {
receipts := *packet.Res.(*eth.ReceiptsResponse)
receipts := *packet.Res.(*eth.ReceiptsRLPResponse)
hashes := packet.Meta.([]common.Hash) // {receipt hashes}
accepted, err := q.queue.DeliverReceipts(peer.id, receipts, hashes)

9
eth/downloader/queue.go
View file

@ -34,6 +34,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
)
const (
@ -69,7 +70,7 @@ type fetchResult struct {
Header *types.Header
Uncles []*types.Header
Transactions types.Transactions
Receipts types.Receipts
Receipts rlp.RawValue
Withdrawals types.Withdrawals
}
@ -318,9 +319,7 @@ func (q *queue) Results(block bool) []*fetchResult {
for _, uncle := range result.Uncles {
size += uncle.Size()
}
for _, receipt := range result.Receipts {
size += receipt.Size()
}
size += common.StorageSize(len(result.Receipts))
for _, tx := range result.Transactions {
size += common.StorageSize(tx.Size())
}
@ -631,7 +630,7 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH
// DeliverReceipts injects a receipt retrieval response into the results queue.
// The method returns the number of transaction receipts accepted from the delivery
// and also wakes any threads waiting for data delivery.
func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt, receiptListHashes []common.Hash) (int, error) {
func (q *queue) DeliverReceipts(id string, receiptList []rlp.RawValue, receiptListHashes []common.Hash) (int, error) {
q.lock.Lock()
defer q.lock.Unlock()

6
eth/downloader/queue_test.go
View file

@ -358,16 +358,16 @@ func XTestDelivery(t *testing.T) {
for {
f, _, _ := q.ReserveReceipts(peer, rand.Intn(50))
if f != nil {
var rcs [][]*types.Receipt
var rcs []types.Receipts
for _, hdr := range f.Headers {
rcs = append(rcs, world.getReceipts(hdr.Number.Uint64()))
}
hasher := trie.NewStackTrie(nil)
hashes := make([]common.Hash, len(rcs))
for i, receipt := range rcs {
hashes[i] = types.DeriveSha(types.Receipts(receipt), hasher)
hashes[i] = types.DeriveSha(receipt, hasher)
}
_, err := q.DeliverReceipts(peer.id, rcs, hashes)
_, err := q.DeliverReceipts(peer.id, types.EncodeBlockReceiptLists(rcs), hashes)
if err != nil {
fmt.Printf("delivered %d receipts %v\n", len(rcs), err)
}

7
eth/filters/filter_system_test.go
View file

@ -80,11 +80,8 @@ func (b *testBackend) GetReceiptsByHash(hash common.Hash) types.Receipts {
return r
}
func (b *testBackend) GetRawReceiptsByHash(hash common.Hash) types.Receipts {
if number := rawdb.ReadHeaderNumber(b.db, hash); number != nil {
return rawdb.ReadRawReceipts(b.db, hash, *number)
}
return nil
func (b *testBackend) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
return rawdb.ReadRawReceipts(b.db, hash, number)
}
func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) {

153
eth/handler.go
View file

@ -18,15 +18,17 @@ package eth
import (
"errors"
"maps"
"math"
"math/big"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@ -48,6 +50,9 @@ const (
// The number is referenced from the size of tx pool.
txChanSize = 4096
// chainHeadChanSize is the size of channel listening to ChainHeadEvent.
chainHeadChanSize = 128
// txMaxBroadcastSize is the max size of a transaction that will be broadcasted.
// All transactions with a higher size will be announced and need to be fetched
// by the peer.
@ -105,7 +110,6 @@ type handlerConfig struct {
type handler struct {
nodeID enode.ID
networkID uint64
forkFilter forkid.Filter // Fork ID filter, constant across the lifetime of the node
snapSync atomic.Bool // Flag whether snap sync is enabled (gets disabled if we already have blocks)
synced atomic.Bool // Flag whether we're considered synchronised (enables transaction processing)
@ -122,6 +126,7 @@ type handler struct {
eventMux *event.TypeMux
txsCh chan core.NewTxsEvent
txsSub event.Subscription
blockRange *blockRangeState
requiredBlocks map[uint64]common.Hash
@ -143,7 +148,6 @@ func newHandler(config *handlerConfig) (*handler, error) {
h := &handler{
nodeID: config.NodeID,
networkID: config.Network,
forkFilter: forkid.NewFilter(config.Chain),
eventMux: config.EventMux,
database: config.Database,
txpool: config.TxPool,
@ -183,7 +187,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
}
// If snap sync is requested but snapshots are disabled, fail loudly
if h.snapSync.Load() && config.Chain.Snapshots() == nil {
if h.snapSync.Load() && (config.Chain.Snapshots() == nil && config.Chain.TrieDB().Scheme() == rawdb.HashScheme) {
return nil, errors.New("snap sync not supported with snapshots disabled")
}
// Construct the downloader (long sync)
@ -256,14 +260,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error {
}
// Execute the Ethereum handshake
var (
genesis = h.chain.Genesis()
head = h.chain.CurrentHeader()
hash = head.Hash()
number = head.Number.Uint64()
)
forkID := forkid.NewID(h.chain.Config(), genesis, number, head.Time)
if err := peer.Handshake(h.networkID, hash, genesis.Hash(), forkID, h.forkFilter); err != nil {
if err := peer.Handshake(h.networkID, h.chain, h.blockRange.currentRange()); err != nil {
peer.Log().Debug("Ethereum handshake failed", "err", err)
return err
}
@ -434,6 +431,11 @@ func (h *handler) Start(maxPeers int) {
h.txsSub = h.txpool.SubscribeTransactions(h.txsCh, false)
go h.txBroadcastLoop()
// broadcast block range
h.wg.Add(1)
h.blockRange = newBlockRangeState(h.chain, h.eventMux)
go h.blockRangeLoop(h.blockRange)
// start sync handlers
h.txFetcher.Start()
@ -444,6 +446,7 @@ func (h *handler) Start(maxPeers int) {
func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop()
h.txFetcher.Stop()
h.downloader.Terminate()
@ -565,3 +568,129 @@ func (h *handler) enableSyncedFeatures() {
h.snapSync.Store(false)
}
}
// blockRangeState holds the state of the block range update broadcasting mechanism.
type blockRangeState struct {
prev eth.BlockRangeUpdatePacket
next atomic.Pointer[eth.BlockRangeUpdatePacket]
headCh chan core.ChainHeadEvent
headSub event.Subscription
syncSub *event.TypeMuxSubscription
}
func newBlockRangeState(chain *core.BlockChain, typeMux *event.TypeMux) *blockRangeState {
headCh := make(chan core.ChainHeadEvent, chainHeadChanSize)
headSub := chain.SubscribeChainHeadEvent(headCh)
syncSub := typeMux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{})
st := &blockRangeState{
headCh: headCh,
headSub: headSub,
syncSub: syncSub,
}
st.update(chain, chain.CurrentBlock())
st.prev = *st.next.Load()
return st
}
// blockRangeBroadcastLoop announces changes in locally-available block range to peers.
// The range to announce is the range that is available in the store, so it's not just
// about imported blocks.
func (h *handler) blockRangeLoop(st *blockRangeState) {
defer h.wg.Done()
for {
select {
case ev := <-st.syncSub.Chan():
if ev == nil {
continue
}
if _, ok := ev.Data.(downloader.StartEvent); ok && h.snapSync.Load() {
h.blockRangeWhileSnapSyncing(st)
}
case <-st.headCh:
st.update(h.chain, h.chain.CurrentBlock())
if st.shouldSend() {
h.broadcastBlockRange(st)
}
case <-st.headSub.Err():
return
}
}
}
// blockRangeWhileSnapSyncing announces block range updates during snap sync.
// Here we poll the CurrentSnapBlock on a timer and announce updates to it.
func (h *handler) blockRangeWhileSnapSyncing(st *blockRangeState) {
tick := time.NewTicker(1 * time.Minute)
defer tick.Stop()
for {
select {
case <-tick.C:
st.update(h.chain, h.chain.CurrentSnapBlock())
if st.shouldSend() {
h.broadcastBlockRange(st)
}
// back to processing head block updates when sync is done
case ev := <-st.syncSub.Chan():
if ev == nil {
continue
}
switch ev.Data.(type) {
case downloader.FailedEvent, downloader.DoneEvent:
return
}
// ignore head updates, but exit when the subscription ends
case <-st.headCh:
case <-st.headSub.Err():
return
}
}
}
// broadcastBlockRange sends a range update when one is due.
func (h *handler) broadcastBlockRange(state *blockRangeState) {
h.peers.lock.Lock()
peerlist := slices.Collect(maps.Values(h.peers.peers))
h.peers.lock.Unlock()
if len(peerlist) == 0 {
return
}
msg := state.currentRange()
log.Debug("Sending BlockRangeUpdate", "peers", len(peerlist), "earliest", msg.EarliestBlock, "latest", msg.LatestBlock)
for _, p := range peerlist {
p.SendBlockRangeUpdate(msg)
}
state.prev = *state.next.Load()
}
// update assigns the values of the next block range update from the chain.
func (st *blockRangeState) update(chain *core.BlockChain, latest *types.Header) {
earliest, _ := chain.HistoryPruningCutoff()
st.next.Store(&eth.BlockRangeUpdatePacket{
EarliestBlock: min(latest.Number.Uint64(), earliest),
LatestBlock: latest.Number.Uint64(),
LatestBlockHash: latest.Hash(),
})
}
// shouldSend decides whether it is time to send a block range update. We don't want to
// send these updates constantly, so they will usually only be sent every 32 blocks.
// However, there is a special case: if the range would move back, i.e. due to SetHead, we
// want to send it immediately.
func (st *blockRangeState) shouldSend() bool {
next := st.next.Load()
return next.LatestBlock < st.prev.LatestBlock ||
next.LatestBlock-st.prev.LatestBlock >= 32
}
func (st *blockRangeState) stop() {
st.syncSub.Unsubscribe()
st.headSub.Unsubscribe()
}
// currentRange returns the current block range.
// This is safe to call from any goroutine.
func (st *blockRangeState) currentRange() eth.BlockRangeUpdatePacket {
return *st.next.Load()
}

13
eth/handler_eth_test.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
@ -257,11 +256,7 @@ func testRecvTransactions(t *testing.T, protocol uint) {
return eth.Handle((*ethHandler)(handler.handler), peer)
})
// Run the handshake locally to avoid spinning up a source handler
var (
genesis = handler.chain.Genesis()
head = handler.chain.CurrentBlock()
)
if err := src.Handshake(1, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil {
if err := src.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil {
t.Fatalf("failed to run protocol handshake")
}
// Send the transaction to the sink and verify that it's added to the tx pool
@ -316,11 +311,7 @@ func testSendTransactions(t *testing.T, protocol uint) {
return eth.Handle((*ethHandler)(handler.handler), peer)
})
// Run the handshake locally to avoid spinning up a source handler
var (
genesis = handler.chain.Genesis()
head = handler.chain.CurrentBlock()
)
if err := sink.Handshake(1, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil {
if err := sink.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil {
t.Fatalf("failed to run protocol handshake")
}
// After the handshake completes, the source handler should stream the sink

18
eth/peer.go
View file

@ -17,6 +17,7 @@
package eth
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap"
)
@ -25,6 +26,13 @@ import (
// about a connected peer.
type ethPeerInfo struct {
Version uint `json:"version"` // Ethereum protocol version negotiated
*peerBlockRange
}
type peerBlockRange struct {
Earliest uint64 `json:"earliestBlock"`
Latest uint64 `json:"latestBlock"`
LatestHash common.Hash `json:"latestBlockHash"`
}
// ethPeer is a wrapper around eth.Peer to maintain a few extra metadata.
@ -35,10 +43,16 @@ type ethPeer struct {
// info gathers and returns some `eth` protocol metadata known about a peer.
func (p *ethPeer) info() *ethPeerInfo {
return &ethPeerInfo{
Version: p.Version(),
info := &ethPeerInfo{Version: p.Version()}
if br := p.BlockRange(); br != nil {
info.peerBlockRange = &peerBlockRange{
Earliest: br.EarliestBlock,
Latest: br.LatestBlock,
LatestHash: br.LatestBlockHash,
}
}
return info
}
// snapPeerInfo represents a short summary of the `snap` sub-protocol metadata known
// about a connected peer.

27
eth/protocols/eth/handler.go
View file

@ -175,12 +175,26 @@ var eth68 = map[uint64]msgHandler{
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts,
ReceiptsMsg: handleReceipts,
GetReceiptsMsg: handleGetReceipts68,
ReceiptsMsg: handleReceipts[*ReceiptList68],
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
}
var eth69 = map[uint64]msgHandler{
TransactionsMsg: handleTransactions,
NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes,
GetBlockHeadersMsg: handleGetBlockHeaders,
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts69,
ReceiptsMsg: handleReceipts[*ReceiptList69],
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
BlockRangeUpdateMsg: handleBlockRangeUpdate,
}
// handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error {
@ -194,7 +208,14 @@ func handleMessage(backend Backend, peer *Peer) error {
}
defer msg.Discard()
var handlers = eth68
var handlers map[uint64]msgHandler
if peer.version == ETH68 {
handlers = eth68
} else if peer.version == ETH69 {
handlers = eth69
} else {
return fmt.Errorf("unknown eth protocol version: %v", peer.version)
}
// Track the amount of time it takes to serve the request and run the handler
if metrics.Enabled() {

15
eth/protocols/eth/handler_test.go
View file

@ -529,22 +529,23 @@ func testGetBlockReceipts(t *testing.T, protocol uint) {
// Collect the hashes to request, and the response to expect
var (
hashes []common.Hash
receipts [][]*types.Receipt
receipts []*ReceiptList68
)
for i := uint64(0); i <= backend.chain.CurrentBlock().Number.Uint64(); i++ {
block := backend.chain.GetBlockByNumber(i)
hashes = append(hashes, block.Hash())
receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash()))
trs := backend.chain.GetReceiptsByHash(block.Hash())
receipts = append(receipts, NewReceiptList68(trs))
}
// Send the hash request and verify the response
p2p.Send(peer.app, GetReceiptsMsg, &GetReceiptsPacket{
RequestId: 123,
GetReceiptsRequest: hashes,
})
if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket{
if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket[*ReceiptList68]{
RequestId: 123,
ReceiptsResponse: receipts,
List: receipts,
}); err != nil {
t.Errorf("receipts mismatch: %v", err)
}
@ -612,10 +613,10 @@ func setup() (*testBackend, *testPeer) {
}
func FuzzEthProtocolHandlers(f *testing.F) {
handlers := eth68
handlers := eth69
backend, peer := setup()
f.Fuzz(func(t *testing.T, code byte, msg []byte) {
handler := handlers[uint64(code)%protocolLengths[ETH68]]
handler := handlers[uint64(code)%protocolLengths[ETH69]]
if handler == nil {
return
}

142
eth/protocols/eth/handlers.go
View file

@ -20,20 +20,25 @@ import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// requestTracker is a singleton tracker for eth/66 and newer request times.
var requestTracker = tracker.New(ProtocolName, 5*time.Minute)
func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error {
// Decode the complex header query
var query GetBlockHeadersPacket
if err := msg.Decode(&query); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
response := ServiceGetBlockHeadersQuery(backend.Chain(), query.GetBlockHeadersRequest, peer)
return peer.ReplyBlockHeadersRLP(query.RequestId, response)
@ -216,7 +221,7 @@ func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error {
// Decode the block body retrieval message
var query GetBlockBodiesPacket
if err := msg.Decode(&query); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
response := ServiceGetBlockBodiesQuery(backend.Chain(), query.GetBlockBodiesRequest)
return peer.ReplyBlockBodiesRLP(query.RequestId, response)
@ -243,19 +248,29 @@ func ServiceGetBlockBodiesQuery(chain *core.BlockChain, query GetBlockBodiesRequ
return bodies
}
func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error {
func handleGetReceipts68(backend Backend, msg Decoder, peer *Peer) error {
// Decode the block receipts retrieval message
var query GetReceiptsPacket
if err := msg.Decode(&query); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
response := ServiceGetReceiptsQuery(backend.Chain(), query.GetReceiptsRequest)
response := ServiceGetReceiptsQuery68(backend.Chain(), query.GetReceiptsRequest)
return peer.ReplyReceiptsRLP(query.RequestId, response)
}
// ServiceGetReceiptsQuery assembles the response to a receipt query. It is
func handleGetReceipts69(backend Backend, msg Decoder, peer *Peer) error {
// Decode the block receipts retrieval message
var query GetReceiptsPacket
if err := msg.Decode(&query); err != nil {
return err
}
response := serviceGetReceiptsQuery69(backend.Chain(), query.GetReceiptsRequest)
return peer.ReplyReceiptsRLP(query.RequestId, response)
}
// ServiceGetReceiptsQuery68 assembles the response to a receipt query. It is
// exposed to allow external packages to test protocol behavior.
func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue {
func ServiceGetReceiptsQuery68(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue {
// Gather state data until the fetch or network limits is reached
var (
bytes int
@ -267,19 +282,62 @@ func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) [
break
}
// Retrieve the requested block's receipts
results := chain.GetReceiptsByHash(hash)
results := chain.GetReceiptsRLP(hash)
if results == nil {
if header := chain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
continue
}
}
// If known, encode and queue for response packet
if encoded, err := rlp.EncodeToBytes(results); err != nil {
log.Error("Failed to encode receipt", "err", err)
} else {
receipts = append(receipts, encoded)
bytes += len(encoded)
body := chain.GetBodyRLP(hash)
if body == nil {
continue
}
var err error
results, err = blockReceiptsToNetwork68(results, body)
if err != nil {
log.Error("Error in block receipts conversion", "hash", hash, "err", err)
continue
}
}
receipts = append(receipts, results)
bytes += len(results)
}
return receipts
}
// serviceGetReceiptsQuery69 assembles the response to a receipt query.
// It does not send the bloom filters for the receipts
func serviceGetReceiptsQuery69(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue {
// Gather state data until the fetch or network limits is reached
var (
bytes int
receipts []rlp.RawValue
)
for lookups, hash := range query {
if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe ||
lookups >= 2*maxReceiptsServe {
break
}
// Retrieve the requested block's receipts
results := chain.GetReceiptsRLP(hash)
if results == nil {
if header := chain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
continue
}
} else {
body := chain.GetBodyRLP(hash)
if body == nil {
continue
}
var err error
results, err = blockReceiptsToNetwork69(results, body)
if err != nil {
log.Error("Error in block receipts conversion", "hash", hash, "err", err)
continue
}
}
receipts = append(receipts, results)
bytes += len(results)
}
return receipts
}
@ -296,7 +354,7 @@ func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error {
// A batch of headers arrived to one of our previous requests
res := new(BlockHeadersPacket)
if err := msg.Decode(res); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
metadata := func() interface{} {
hashes := make([]common.Hash, len(res.BlockHeadersRequest))
@ -316,7 +374,7 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error {
// A batch of block bodies arrived to one of our previous requests
res := new(BlockBodiesPacket)
if err := msg.Decode(res); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
metadata := func() interface{} {
var (
@ -341,24 +399,35 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error {
}, metadata)
}
func handleReceipts(backend Backend, msg Decoder, peer *Peer) error {
func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) error {
// A batch of receipts arrived to one of our previous requests
res := new(ReceiptsPacket)
res := new(ReceiptsPacket[L])
if err := msg.Decode(res); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
// Assign temporary hashing buffer to each list item, the same buffer is shared
// between all receipt list instances.
buffers := new(receiptListBuffers)
for i := range res.List {
res.List[i].setBuffers(buffers)
}
metadata := func() interface{} {
hasher := trie.NewStackTrie(nil)
hashes := make([]common.Hash, len(res.ReceiptsResponse))
for i, receipt := range res.ReceiptsResponse {
hashes[i] = types.DeriveSha(types.Receipts(receipt), hasher)
hashes := make([]common.Hash, len(res.List))
for i := range res.List {
hashes[i] = types.DeriveSha(res.List[i], hasher)
}
return hashes
}
var enc ReceiptsRLPResponse
for i := range res.List {
enc = append(enc, res.List[i].EncodeForStorage())
}
return peer.dispatchResponse(&Response{
id: res.RequestId,
code: ReceiptsMsg,
Res: &res.ReceiptsResponse,
Res: &enc,
}, metadata)
}
@ -370,10 +439,10 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
}
ann := new(NewPooledTransactionHashesPacket)
if err := msg.Decode(ann); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
return fmt.Errorf("%w: message %v: invalid len of fields: %v %v %v", errDecode, msg, len(ann.Hashes), len(ann.Types), len(ann.Sizes))
return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
}
// Schedule all the unknown hashes for retrieval
for _, hash := range ann.Hashes {
@ -386,7 +455,7 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
// Decode the pooled transactions retrieval message
var query GetPooledTransactionsPacket
if err := msg.Decode(&query); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
@ -423,12 +492,12 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error {
// Transactions can be processed, parse all of them and deliver to the pool
var txs TransactionsPacket
if err := msg.Decode(&txs); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
for i, tx := range txs {
// Validate and mark the remote transaction
if tx == nil {
return fmt.Errorf("%w: transaction %d is nil", errDecode, i)
return fmt.Errorf("Transactions: transaction %d is nil", i)
}
peer.markTransaction(tx.Hash())
}
@ -443,12 +512,12 @@ func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error {
// Transactions can be processed, parse all of them and deliver to the pool
var txs PooledTransactionsPacket
if err := msg.Decode(&txs); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
return err
}
for i, tx := range txs.PooledTransactionsResponse {
// Validate and mark the remote transaction
if tx == nil {
return fmt.Errorf("%w: transaction %d is nil", errDecode, i)
return fmt.Errorf("PooledTransactions: transaction %d is nil", i)
}
peer.markTransaction(tx.Hash())
}
@ -456,3 +525,16 @@ func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error {
return backend.Handle(peer, &txs.PooledTransactionsResponse)
}
func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
var update BlockRangeUpdatePacket
if err := msg.Decode(&update); err != nil {
return err
}
if err := update.Validate(); err != nil {
return err
}
// We don't do anything with these messages for now, just store them on the peer.
peer.lastRange.Store(&update)
return nil
}

166
eth/protocols/eth/handshake.go
View file

@ -19,10 +19,10 @@ package eth
import (
"errors"
"fmt"
"math/big"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
@ -36,44 +36,122 @@ const (
// Handshake executes the eth protocol handshake, negotiating version number,
// network IDs, difficulties, head and genesis blocks.
func (p *Peer) Handshake(network uint64, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error {
// Send out own handshake in a new thread
func (p *Peer) Handshake(networkID uint64, chain *core.BlockChain, rangeMsg BlockRangeUpdatePacket) error {
switch p.version {
case ETH69:
return p.handshake69(networkID, chain, rangeMsg)
case ETH68:
return p.handshake68(networkID, chain)
default:
return errors.New("unsupported protocol version")
}
}
func (p *Peer) handshake68(networkID uint64, chain *core.BlockChain) error {
var (
genesis = chain.Genesis()
latest = chain.CurrentBlock()
forkID = forkid.NewID(chain.Config(), genesis, latest.Number.Uint64(), latest.Time)
forkFilter = forkid.NewFilter(chain)
)
errc := make(chan error, 2)
var status StatusPacket // safe to read after two values have been received from errc
go func() {
errc <- p2p.Send(p.rw, StatusMsg, &StatusPacket{
pkt := &StatusPacket68{
ProtocolVersion: uint32(p.version),
NetworkID: network,
TD: new(big.Int), // unknown for post-merge tail=pruned networks
Head: head,
Genesis: genesis,
NetworkID: networkID,
Head: latest.Hash(),
Genesis: genesis.Hash(),
ForkID: forkID,
})
}
errc <- p2p.Send(p.rw, StatusMsg, pkt)
}()
var status StatusPacket68 // safe to read after two values have been received from errc
go func() {
errc <- p.readStatus(network, &status, genesis, forkFilter)
errc <- p.readStatus68(networkID, &status, genesis.Hash(), forkFilter)
}()
timeout := time.NewTimer(handshakeTimeout)
defer timeout.Stop()
for i := 0; i < 2; i++ {
select {
case err := <-errc:
if err != nil {
markError(p, err)
return waitForHandshake(errc, p)
}
func (p *Peer) readStatus68(networkID uint64, status *StatusPacket68, genesis common.Hash, forkFilter forkid.Filter) error {
if err := p.readStatusMsg(status); err != nil {
return err
}
case <-timeout.C:
markError(p, p2p.DiscReadTimeout)
return p2p.DiscReadTimeout
if status.NetworkID != networkID {
return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, networkID)
}
if uint(status.ProtocolVersion) != p.version {
return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version)
}
if status.Genesis != genesis {
return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis)
}
if err := forkFilter(status.ForkID); err != nil {
return fmt.Errorf("%w: %v", errForkIDRejected, err)
}
return nil
}
// readStatus reads the remote handshake message.
func (p *Peer) readStatus(network uint64, status *StatusPacket, genesis common.Hash, forkFilter forkid.Filter) error {
func (p *Peer) handshake69(networkID uint64, chain *core.BlockChain, rangeMsg BlockRangeUpdatePacket) error {
var (
genesis = chain.Genesis()
latest = chain.CurrentBlock()
forkID = forkid.NewID(chain.Config(), genesis, latest.Number.Uint64(), latest.Time)
forkFilter = forkid.NewFilter(chain)
)
errc := make(chan error, 2)
go func() {
pkt := &StatusPacket69{
ProtocolVersion: uint32(p.version),
NetworkID: networkID,
Genesis: genesis.Hash(),
ForkID: forkID,
EarliestBlock: rangeMsg.EarliestBlock,
LatestBlock: rangeMsg.LatestBlock,
LatestBlockHash: rangeMsg.LatestBlockHash,
}
errc <- p2p.Send(p.rw, StatusMsg, pkt)
}()
var status StatusPacket69 // safe to read after two values have been received from errc
go func() {
errc <- p.readStatus69(networkID, &status, genesis.Hash(), forkFilter)
}()
return waitForHandshake(errc, p)
}
func (p *Peer) readStatus69(networkID uint64, status *StatusPacket69, genesis common.Hash, forkFilter forkid.Filter) error {
if err := p.readStatusMsg(status); err != nil {
return err
}
if status.NetworkID != networkID {
return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, networkID)
}
if uint(status.ProtocolVersion) != p.version {
return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version)
}
if status.Genesis != genesis {
return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis)
}
if err := forkFilter(status.ForkID); err != nil {
return fmt.Errorf("%w: %v", errForkIDRejected, err)
}
// Handle initial block range.
initRange := &BlockRangeUpdatePacket{
EarliestBlock: status.EarliestBlock,
LatestBlock: status.LatestBlock,
LatestBlockHash: status.LatestBlockHash,
}
if err := initRange.Validate(); err != nil {
return fmt.Errorf("%w: %v", errInvalidBlockRange, err)
}
p.lastRange.Store(initRange)
return nil
}
// readStatusMsg reads the first message on the connection.
func (p *Peer) readStatusMsg(dst any) error {
msg, err := p.rw.ReadMsg()
if err != nil {
return err
@ -84,21 +162,26 @@ func (p *Peer) readStatus(network uint64, status *StatusPacket, genesis common.H
if msg.Size > maxMessageSize {
return fmt.Errorf("%w: %v > %v", errMsgTooLarge, msg.Size, maxMessageSize)
}
// Decode the handshake and make sure everything matches
if err := msg.Decode(&status); err != nil {
return fmt.Errorf("%w: message %v: %v", errDecode, msg, err)
if err := msg.Decode(dst); err != nil {
return err
}
if status.NetworkID != network {
return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, network)
return nil
}
if uint(status.ProtocolVersion) != p.version {
return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version)
func waitForHandshake(errc <-chan error, p *Peer) error {
timeout := time.NewTimer(handshakeTimeout)
defer timeout.Stop()
for range 2 {
select {
case err := <-errc:
if err != nil {
markError(p, err)
return err
}
if status.Genesis != genesis {
return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis)
case <-timeout.C:
markError(p, p2p.DiscReadTimeout)
return p2p.DiscReadTimeout
}
if err := forkFilter(status.ForkID); err != nil {
return fmt.Errorf("%w: %v", errForkIDRejected, err)
}
return nil
}
@ -124,3 +207,14 @@ func markError(p *Peer, err error) {
m.peerError.Mark(1)
}
}
// Validate checks basic validity of a block range announcement.
func (p *BlockRangeUpdatePacket) Validate() error {
if p.EarliestBlock > p.LatestBlock {
return errors.New("earliest > latest")
}
if p.LatestBlockHash == (common.Hash{}) {
return errors.New("zero latest hash")
}
return nil
}

10
eth/protocols/eth/handshake_test.go
View file

@ -52,19 +52,19 @@ func testHandshake(t *testing.T, protocol uint) {
want: errNoStatusMsg,
},
{
code: StatusMsg, data: StatusPacket{10, 1, new(big.Int), head.Hash(), genesis.Hash(), forkID},
code: StatusMsg, data: StatusPacket68{10, 1, new(big.Int), head.Hash(), genesis.Hash(), forkID},
want: errProtocolVersionMismatch,
},
{
code: StatusMsg, data: StatusPacket{uint32(protocol), 999, new(big.Int), head.Hash(), genesis.Hash(), forkID},
code: StatusMsg, data: StatusPacket68{uint32(protocol), 999, new(big.Int), head.Hash(), genesis.Hash(), forkID},
want: errNetworkIDMismatch,
},
{
code: StatusMsg, data: StatusPacket{uint32(protocol), 1, new(big.Int), head.Hash(), common.Hash{3}, forkID},
code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), common.Hash{3}, forkID},
want: errGenesisMismatch,
},
{
code: StatusMsg, data: StatusPacket{uint32(protocol), 1, new(big.Int), head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}},
code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}},
want: errForkIDRejected,
},
}
@ -80,7 +80,7 @@ func testHandshake(t *testing.T, protocol uint) {
// Send the junk test with one peer, check the handshake failure
go p2p.Send(app, test.code, test.data)
err := peer.Handshake(1, head.Hash(), genesis.Hash(), forkID, forkid.NewFilter(backend.chain))
err := peer.Handshake(1, backend.chain, BlockRangeUpdatePacket{})
if err == nil {
t.Errorf("test %d: protocol returned nil error, want %q", i, test.want)
} else if !errors.Is(err, test.want) {

16
eth/protocols/eth/peer.go
View file

@ -18,6 +18,7 @@ package eth
import (
"math/rand"
"sync/atomic"
mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common"
@ -47,6 +48,7 @@ type Peer struct {
*p2p.Peer // The embedded P2P package peer
rw p2p.MsgReadWriter // Input/output streams for snap
version uint // Protocol version negotiated
lastRange atomic.Pointer[BlockRangeUpdatePacket]
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
knownTxs *knownCache // Set of transaction hashes known to be known by this peer
@ -102,6 +104,12 @@ func (p *Peer) Version() uint {
return p.version
}
// BlockRange returns the latest announced block range.
// This will be nil for peers below protocol version eth/69.
func (p *Peer) BlockRange() *BlockRangeUpdatePacket {
return p.lastRange.Load()
}
// KnownTransaction returns whether peer is known to already have a transaction.
func (p *Peer) KnownTransaction(hash common.Hash) bool {
return p.knownTxs.Contains(hash)
@ -343,6 +351,14 @@ func (p *Peer) RequestTxs(hashes []common.Hash) error {
})
}
// SendBlockRangeUpdate sends a notification about our available block range to the peer.
func (p *Peer) SendBlockRangeUpdate(msg BlockRangeUpdatePacket) error {
if p.version < ETH69 {
return nil
}
return p2p.Send(p.rw, BlockRangeUpdateMsg, &msg)
}
// knownCache is a cache for known hashes.
type knownCache struct {
hashes mapset.Set[common.Hash]

59
eth/protocols/eth/protocol.go
View file

@ -31,6 +31,7 @@ import (
// Constants to match up protocol versions and messages
const (
ETH68 = 68
ETH69 = 69
)
// ProtocolName is the official short name of the `eth` protocol used during
@ -39,11 +40,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary).
var ProtocolVersions = []uint{ETH68}
var ProtocolVersions = []uint{ETH69, ETH68}
// protocolLengths are the number of implemented message corresponding to
// different protocol versions.
var protocolLengths = map[uint]uint64{ETH68: 17}
var protocolLengths = map[uint]uint64{ETH68: 17, ETH69: 18}
// maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024
@ -62,17 +63,19 @@ const (
PooledTransactionsMsg = 0x0a
GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10
BlockRangeUpdateMsg = 0x11
)
var (
errNoStatusMsg = errors.New("no status message")
errMsgTooLarge = errors.New("message too long")
errDecode = errors.New("invalid message")
errInvalidMsgCode = errors.New("invalid message code")
errProtocolVersionMismatch = errors.New("protocol version mismatch")
// handshake errors
errNoStatusMsg = errors.New("no status message")
errNetworkIDMismatch = errors.New("network ID mismatch")
errGenesisMismatch = errors.New("genesis mismatch")
errForkIDRejected = errors.New("fork ID rejected")
errInvalidBlockRange = errors.New("invalid block range in status")
)
// Packet represents a p2p message in the `eth` protocol.
@ -82,7 +85,7 @@ type Packet interface {
}
// StatusPacket is the network packet for the status message.
type StatusPacket struct {
type StatusPacket68 struct {
ProtocolVersion uint32
NetworkID uint64
TD *big.Int
@ -91,6 +94,18 @@ type StatusPacket struct {
ForkID forkid.ID
}
// StatusPacket69 is the network packet for the status message.
type StatusPacket69 struct {
ProtocolVersion uint32
NetworkID uint64
Genesis common.Hash
ForkID forkid.ID
// initial available block range
EarliestBlock uint64
LatestBlock uint64
LatestBlockHash common.Hash
}
// NewBlockHashesPacket is the network packet for the block announcements.
type NewBlockHashesPacket []struct {
Hash common.Hash // Hash of one particular block being announced
@ -250,13 +265,21 @@ type GetReceiptsPacket struct {
}
// ReceiptsResponse is the network packet for block receipts distribution.
type ReceiptsResponse [][]*types.Receipt
type ReceiptsResponse []types.Receipts
// ReceiptsList is a type constraint for block receceipt list types.
type ReceiptsList interface {
*ReceiptList68 | *ReceiptList69
setBuffers(*receiptListBuffers)
EncodeForStorage() rlp.RawValue
types.DerivableList
}
// ReceiptsPacket is the network packet for block receipts distribution with
// request ID wrapping.
type ReceiptsPacket struct {
type ReceiptsPacket[L ReceiptsList] struct {
RequestId uint64
ReceiptsResponse
List []L
}
// ReceiptsRLPResponse is used for receipts, when we already have it encoded
@ -304,8 +327,18 @@ type PooledTransactionsRLPPacket struct {
PooledTransactionsRLPResponse
}
func (*StatusPacket) Name() string { return "Status" }
func (*StatusPacket) Kind() byte { return StatusMsg }
// BlockRangeUpdatePacket is an announcement of the node's available block range.
type BlockRangeUpdatePacket struct {
EarliestBlock uint64
LatestBlock uint64
LatestBlockHash common.Hash
}
func (*StatusPacket68) Name() string { return "Status" }
func (*StatusPacket68) Kind() byte { return StatusMsg }
func (*StatusPacket69) Name() string { return "Status" }
func (*StatusPacket69) Kind() byte { return StatusMsg }
func (*NewBlockHashesPacket) Name() string { return "NewBlockHashes" }
func (*NewBlockHashesPacket) Kind() byte { return NewBlockHashesMsg }
@ -342,3 +375,9 @@ func (*GetReceiptsRequest) Kind() byte { return GetReceiptsMsg }
func (*ReceiptsResponse) Name() string { return "Receipts" }
func (*ReceiptsResponse) Kind() byte { return ReceiptsMsg }
func (*ReceiptsRLPResponse) Name() string { return "Receipts" }
func (*ReceiptsRLPResponse) Kind() byte { return ReceiptsMsg }
func (*BlockRangeUpdatePacket) Name() string { return "BlockRangeUpdate" }
func (*BlockRangeUpdatePacket) Kind() byte { return BlockRangeUpdateMsg }

35
eth/protocols/eth/protocol_test.go
View file

@ -75,7 +75,7 @@ func TestEmptyMessages(t *testing.T) {
// All empty messages encodes to the same format
want := common.FromHex("c4820457c0")
for i, msg := range []interface{}{
for i, msg := range []any{
// Headers
GetBlockHeadersPacket{1111, nil},
BlockHeadersPacket{1111, nil},
@ -85,7 +85,6 @@ func TestEmptyMessages(t *testing.T) {
BlockBodiesRLPPacket{1111, nil},
// Receipts
GetReceiptsPacket{1111, nil},
ReceiptsPacket{1111, nil},
// Transactions
GetPooledTransactionsPacket{1111, nil},
PooledTransactionsPacket{1111, nil},
@ -99,7 +98,8 @@ func TestEmptyMessages(t *testing.T) {
BlockBodiesRLPPacket{1111, BlockBodiesRLPResponse([]rlp.RawValue{})},
// Receipts
GetReceiptsPacket{1111, GetReceiptsRequest([]common.Hash{})},
ReceiptsPacket{1111, ReceiptsResponse([][]*types.Receipt{})},
ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{}},
ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{}},
// Transactions
GetPooledTransactionsPacket{1111, GetPooledTransactionsRequest([]common.Hash{})},
PooledTransactionsPacket{1111, PooledTransactionsResponse([]*types.Transaction{})},
@ -168,7 +168,7 @@ func TestMessages(t *testing.T) {
receipts = []*types.Receipt{
{
Status: types.ReceiptStatusFailed,
CumulativeGasUsed: 1,
CumulativeGasUsed: 333,
Logs: []*types.Log{
{
Address: common.BytesToAddress([]byte{0x11}),
@ -176,11 +176,21 @@ func TestMessages(t *testing.T) {
Data: []byte{0x01, 0x00, 0xff},
},
},
TxHash: hashes[0],
ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}),
GasUsed: 111111,
},
{
Status: types.ReceiptStatusSuccessful,
CumulativeGasUsed: 444,
Logs: []*types.Log{
{
Address: common.BytesToAddress([]byte{0x22}),
Topics: []common.Hash{common.HexToHash("05668"), common.HexToHash("9773")},
Data: []byte{0x02, 0x0f, 0x0f, 0x0f, 0x06, 0x08},
},
},
},
}
miniDeriveFields(receipts[0], 0)
miniDeriveFields(receipts[1], 1)
rlpData, err := rlp.EncodeToBytes(receipts)
if err != nil {
t.Fatal(err)
@ -221,12 +231,17 @@ func TestMessages(t *testing.T) {
common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"),
},
{
ReceiptsPacket{1111, ReceiptsResponse([][]*types.Receipt{receipts})},
common.FromHex("f90172820457f9016cf90169f901668001b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ff"),
ReceiptsPacket[*ReceiptList68]{1111, []*ReceiptList68{NewReceiptList68(receipts)}},
common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"),
},
{
// Identical to the eth/68 encoding above.
ReceiptsRLPPacket{1111, ReceiptsRLPResponse([]rlp.RawValue{receiptsRlp})},
common.FromHex("f90172820457f9016cf90169f901668001b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ff"),
common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"),
},
{
ReceiptsPacket[*ReceiptList69]{1111, []*ReceiptList69{NewReceiptList69(receipts)}},
common.FromHex("f8da820457f8d5f8d3f866808082014df85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100fff86901018201bcf862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"),
},
{
GetPooledTransactionsPacket{1111, GetPooledTransactionsRequest(hashes)},

462
eth/protocols/eth/receipt.go Normal file
View file

@ -0,0 +1,462 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"bytes"
"fmt"
"io"
"iter"
"math/big"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
)
// This is just a sanity limit for the size of a single receipt.
const maxReceiptSize = 16 * 1024 * 1024
// Receipt is the representation of receipts for networking purposes.
type Receipt struct {
TxType byte
PostStateOrStatus []byte
GasUsed uint64
Logs rlp.RawValue
}
func newReceipt(tr *types.Receipt) Receipt {
r := Receipt{TxType: tr.Type, GasUsed: tr.CumulativeGasUsed}
if tr.PostState != nil {
r.PostStateOrStatus = tr.PostState
} else {
r.PostStateOrStatus = new(big.Int).SetUint64(tr.Status).Bytes()
}
r.Logs, _ = rlp.EncodeToBytes(tr.Logs)
return r
}
// decode68 parses a receipt in the eth/68 network encoding.
func (r *Receipt) decode68(buf *receiptListBuffers, s *rlp.Stream) error {
k, size, err := s.Kind()
if err != nil {
return err
}
*r = Receipt{}
if k == rlp.List {
// Legacy receipt.
return r.decodeInnerList(s, false, true)
}
// Typed receipt.
if size < 2 || size > maxReceiptSize {
return fmt.Errorf("invalid receipt size %d", size)
}
buf.tmp.Reset()
buf.tmp.Grow(int(size))
payload := buf.tmp.Bytes()[:int(size)]
if err := s.ReadBytes(payload); err != nil {
return err
}
r.TxType = payload[0]
s2 := rlp.NewStream(bytes.NewReader(payload[1:]), 0)
return r.decodeInnerList(s2, false, true)
}
// decode69 parses a receipt in the eth/69 network encoding.
func (r *Receipt) decode69(s *rlp.Stream) error {
*r = Receipt{}
return r.decodeInnerList(s, true, false)
}
// decodeDatabase parses a receipt in the basic database encoding.
func (r *Receipt) decodeDatabase(txType byte, s *rlp.Stream) error {
*r = Receipt{TxType: txType}
return r.decodeInnerList(s, false, false)
}
func (r *Receipt) decodeInnerList(s *rlp.Stream, readTxType, readBloom bool) error {
_, err := s.List()
if err != nil {
return err
}
if readTxType {
r.TxType, err = s.Uint8()
if err != nil {
return fmt.Errorf("invalid txType: %w", err)
}
}
r.PostStateOrStatus, err = s.Bytes()
if err != nil {
return fmt.Errorf("invalid postStateOrStatus: %w", err)
}
r.GasUsed, err = s.Uint64()
if err != nil {
return fmt.Errorf("invalid gasUsed: %w", err)
}
if readBloom {
var b types.Bloom
if err := s.ReadBytes(b[:]); err != nil {
return fmt.Errorf("invalid bloom: %v", err)
}
}
r.Logs, err = s.Raw()
if err != nil {
return fmt.Errorf("invalid logs: %w", err)
}
return s.ListEnd()
}
// encodeForStorage produces the the storage encoding, i.e. the result matches
// the RLP encoding of types.ReceiptForStorage.
func (r *Receipt) encodeForStorage(w *rlp.EncoderBuffer) {
list := w.List()
w.WriteBytes(r.PostStateOrStatus)
w.WriteUint64(r.GasUsed)
w.Write(r.Logs)
w.ListEnd(list)
}
// encodeForNetwork68 produces the eth/68 network protocol encoding of a receipt.
// Note this recomputes the bloom filter of the receipt.
func (r *Receipt) encodeForNetwork68(buf *receiptListBuffers, w *rlp.EncoderBuffer) {
writeInner := func(w *rlp.EncoderBuffer) {
list := w.List()
w.WriteBytes(r.PostStateOrStatus)
w.WriteUint64(r.GasUsed)
bloom := r.bloom(&buf.bloom)
w.WriteBytes(bloom[:])
w.Write(r.Logs)
w.ListEnd(list)
}
if r.TxType == 0 {
writeInner(w)
} else {
buf.tmp.Reset()
buf.tmp.WriteByte(r.TxType)
buf.enc.Reset(&buf.tmp)
writeInner(&buf.enc)
buf.enc.Flush()
w.WriteBytes(buf.tmp.Bytes())
}
}
// encodeForNetwork69 produces the eth/69 network protocol encoding of a receipt.
func (r *Receipt) encodeForNetwork69(w *rlp.EncoderBuffer) {
list := w.List()
w.WriteUint64(uint64(r.TxType))
w.WriteBytes(r.PostStateOrStatus)
w.WriteUint64(r.GasUsed)
w.Write(r.Logs)
w.ListEnd(list)
}
// encodeForHash encodes a receipt for the block receiptsRoot derivation.
func (r *Receipt) encodeForHash(buf *receiptListBuffers, out *bytes.Buffer) {
// For typed receipts, add the tx type.
if r.TxType != 0 {
out.WriteByte(r.TxType)
}
// Encode list = [postStateOrStatus, gasUsed, bloom, logs].
w := &buf.enc
w.Reset(out)
l := w.List()
w.WriteBytes(r.PostStateOrStatus)
w.WriteUint64(r.GasUsed)
bloom := r.bloom(&buf.bloom)
w.WriteBytes(bloom[:])
w.Write(r.Logs)
w.ListEnd(l)
w.Flush()
}
// bloom computes the bloom filter of the receipt.
// Note this doesn't check the validity of encoding, and will produce an invalid filter
// for invalid input. This is acceptable for the purpose of this function, which is
// recomputing the receipt hash.
func (r *Receipt) bloom(buffer *[6]byte) types.Bloom {
var b types.Bloom
logsIter, err := rlp.NewListIterator(r.Logs)
if err != nil {
return b
}
for logsIter.Next() {
log, _, _ := rlp.SplitList(logsIter.Value())
address, log, _ := rlp.SplitString(log)
b.AddWithBuffer(address, buffer)
topicsIter, err := rlp.NewListIterator(log)
if err != nil {
return b
}
for topicsIter.Next() {
topic, _, _ := rlp.SplitString(topicsIter.Value())
b.AddWithBuffer(topic, buffer)
}
}
return b
}
type receiptListBuffers struct {
enc rlp.EncoderBuffer
bloom [6]byte
tmp bytes.Buffer
}
func initBuffers(buf **receiptListBuffers) {
if *buf == nil {
*buf = new(receiptListBuffers)
}
}
// encodeForStorage encodes a list of receipts for the database.
func (buf *receiptListBuffers) encodeForStorage(rs []Receipt) rlp.RawValue {
var out bytes.Buffer
w := &buf.enc
w.Reset(&out)
outer := w.List()
for _, receipts := range rs {
receipts.encodeForStorage(w)
}
w.ListEnd(outer)
w.Flush()
return out.Bytes()
}
// ReceiptList68 is a block receipt list as downloaded by eth/68.
// This also implements types.DerivableList for validation purposes.
type ReceiptList68 struct {
buf *receiptListBuffers
items []Receipt
}
// NewReceiptList68 creates a receipt list.
// This is slow, and exists for testing purposes.
func NewReceiptList68(trs []*types.Receipt) *ReceiptList68 {
rl := &ReceiptList68{items: make([]Receipt, len(trs))}
for i, tr := range trs {
rl.items[i] = newReceipt(tr)
}
return rl
}
func blockReceiptsToNetwork68(blockReceipts, blockBody rlp.RawValue) ([]byte, error) {
txTypesIter, err := txTypesInBody(blockBody)
if err != nil {
return nil, fmt.Errorf("invalid block body: %v", err)
}
nextTxType, stopTxTypes := iter.Pull(txTypesIter)
defer stopTxTypes()
var (
out bytes.Buffer
buf receiptListBuffers
)
blockReceiptIter, _ := rlp.NewListIterator(blockReceipts)
innerReader := bytes.NewReader(nil)
innerStream := rlp.NewStream(innerReader, 0)
w := rlp.NewEncoderBuffer(&out)
outer := w.List()
for i := 0; blockReceiptIter.Next(); i++ {
content := blockReceiptIter.Value()
innerReader.Reset(content)
innerStream.Reset(innerReader, uint64(len(content)))
var r Receipt
txType, _ := nextTxType()
if err := r.decodeDatabase(txType, innerStream); err != nil {
return nil, fmt.Errorf("invalid database receipt %d: %v", i, err)
}
r.encodeForNetwork68(&buf, &w)
}
w.ListEnd(outer)
w.Flush()
return out.Bytes(), nil
}
// setBuffers implements ReceiptsList.
func (rl *ReceiptList68) setBuffers(buf *receiptListBuffers) {
rl.buf = buf
}
// EncodeForStorage encodes the receipts for storage into the database.
func (rl *ReceiptList68) EncodeForStorage() rlp.RawValue {
initBuffers(&rl.buf)
return rl.buf.encodeForStorage(rl.items)
}
// Len implements types.DerivableList.
func (rl *ReceiptList68) Len() int {
return len(rl.items)
}
// EncodeIndex implements types.DerivableList.
func (rl *ReceiptList68) EncodeIndex(i int, out *bytes.Buffer) {
initBuffers(&rl.buf)
rl.items[i].encodeForHash(rl.buf, out)
}
// DecodeRLP decodes a list of receipts from the network format.
func (rl *ReceiptList68) DecodeRLP(s *rlp.Stream) error {
initBuffers(&rl.buf)
if _, err := s.List(); err != nil {
return err
}
for i := 0; s.MoreDataInList(); i++ {
var item Receipt
err := item.decode68(rl.buf, s)
if err != nil {
return fmt.Errorf("receipt %d: %v", i, err)
}
rl.items = append(rl.items, item)
}
return s.ListEnd()
}
// EncodeRLP encodes the list into the network format of eth/68.
func (rl *ReceiptList68) EncodeRLP(_w io.Writer) error {
initBuffers(&rl.buf)
w := rlp.NewEncoderBuffer(_w)
outer := w.List()
for i := range rl.items {
rl.items[i].encodeForNetwork68(rl.buf, &w)
}
w.ListEnd(outer)
return w.Flush()
}
// ReceiptList69 is the block receipt list as downloaded by eth/69.
// This implements types.DerivableList for validation purposes.
type ReceiptList69 struct {
buf *receiptListBuffers
items []Receipt
}
// NewReceiptList69 creates a receipt list.
// This is slow, and exists for testing purposes.
func NewReceiptList69(trs []*types.Receipt) *ReceiptList69 {
rl := &ReceiptList69{items: make([]Receipt, len(trs))}
for i, tr := range trs {
rl.items[i] = newReceipt(tr)
}
return rl
}
// setBuffers implements ReceiptsList.
func (rl *ReceiptList69) setBuffers(buf *receiptListBuffers) {
rl.buf = buf
}
// EncodeForStorage encodes the receipts for storage into the database.
func (rl *ReceiptList69) EncodeForStorage() rlp.RawValue {
initBuffers(&rl.buf)
return rl.buf.encodeForStorage(rl.items)
}
// Len implements types.DerivableList.
func (rl *ReceiptList69) Len() int {
return len(rl.items)
}
// EncodeIndex implements types.DerivableList.
func (rl *ReceiptList69) EncodeIndex(i int, out *bytes.Buffer) {
initBuffers(&rl.buf)
rl.items[i].encodeForHash(rl.buf, out)
}
// DecodeRLP decodes a list receipts from the network format.
func (rl *ReceiptList69) DecodeRLP(s *rlp.Stream) error {
if _, err := s.List(); err != nil {
return err
}
for i := 0; s.MoreDataInList(); i++ {
var item Receipt
err := item.decode69(s)
if err != nil {
return fmt.Errorf("receipt %d: %v", i, err)
}
rl.items = append(rl.items, item)
}
return s.ListEnd()
}
// EncodeRLP encodes the list into the network format of eth/69.
func (rl *ReceiptList69) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
outer := w.List()
for i := range rl.items {
rl.items[i].encodeForNetwork69(&w)
}
w.ListEnd(outer)
return w.Flush()
}
// blockReceiptsToNetwork69 takes a slice of rlp-encoded receipts, and transactions,
// and applies the type-encoding on the receipts (for non-legacy receipts).
// e.g. for non-legacy receipts: receipt-data -> {tx-type || receipt-data}
func blockReceiptsToNetwork69(blockReceipts, blockBody rlp.RawValue) ([]byte, error) {
txTypesIter, err := txTypesInBody(blockBody)
if err != nil {
return nil, fmt.Errorf("invalid block body: %v", err)
}
nextTxType, stopTxTypes := iter.Pull(txTypesIter)
defer stopTxTypes()
var (
out bytes.Buffer
enc = rlp.NewEncoderBuffer(&out)
it, _ = rlp.NewListIterator(blockReceipts)
)
outer := enc.List()
for i := 0; it.Next(); i++ {
txType, _ := nextTxType()
content, _, _ := rlp.SplitList(it.Value())
receiptList := enc.List()
enc.WriteUint64(uint64(txType))
enc.Write(content)
enc.ListEnd(receiptList)
}
enc.ListEnd(outer)
enc.Flush()
return out.Bytes(), nil
}
// txTypesInBody parses the transactions list of an encoded block body, returning just the types.
func txTypesInBody(body rlp.RawValue) (iter.Seq[byte], error) {
bodyFields, _, err := rlp.SplitList(body)
if err != nil {
return nil, err
}
txsIter, err := rlp.NewListIterator(bodyFields)
if err != nil {
return nil, err
}
return func(yield func(byte) bool) {
for txsIter.Next() {
var txType byte
switch k, content, _, _ := rlp.Split(txsIter.Value()); k {
case rlp.List:
txType = 0
case rlp.String:
if len(content) > 0 {
txType = content[0]
}
}
if !yield(txType) {
return
}
}
}, nil
}

158
eth/protocols/eth/receipt_test.go Normal file
View file

@ -0,0 +1,158 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"bytes"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// miniDeriveFields derives the necessary receipt fields to make types.DeriveSha work.
func miniDeriveFields(r *types.Receipt, txType byte) {
r.Type = txType
r.Bloom = types.CreateBloom(r)
}
var receiptsTestLogs1 = []*types.Log{{Address: common.Address{1}, Topics: []common.Hash{{1}}}}
var receiptsTestLogs2 = []*types.Log{
{Address: common.Address{2}, Topics: []common.Hash{{21}, {22}}, Data: []byte{2, 2, 32, 32}},
{Address: common.Address{3}, Topics: []common.Hash{{31}, {32}}, Data: []byte{3, 3, 32, 32}},
}
var receiptsTests = []struct {
input []types.ReceiptForStorage
txs []*types.Transaction
root common.Hash
}{
{
input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, Status: 1, Logs: nil}},
txs: []*types.Transaction{types.NewTx(&types.LegacyTx{})},
},
{
input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, Status: 1, Logs: nil}},
txs: []*types.Transaction{types.NewTx(&types.DynamicFeeTx{})},
},
{
input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, Status: 1, Logs: nil}},
txs: []*types.Transaction{types.NewTx(&types.AccessListTx{})},
},
{
input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, Status: 1, Logs: receiptsTestLogs1}},
txs: []*types.Transaction{types.NewTx(&types.LegacyTx{})},
},
{
input: []types.ReceiptForStorage{{CumulativeGasUsed: 555, Status: 1, Logs: receiptsTestLogs2}},
txs: []*types.Transaction{types.NewTx(&types.AccessListTx{})},
},
}
func init() {
for i := range receiptsTests {
// derive basic fields
for j := range receiptsTests[i].input {
r := (*types.Receipt)(&receiptsTests[i].input[j])
txType := receiptsTests[i].txs[j].Type()
miniDeriveFields(r, txType)
}
// compute expected root
receipts := make(types.Receipts, len(receiptsTests[i].input))
for j, sr := range receiptsTests[i].input {
r := types.Receipt(sr)
receipts[j] = &r
}
receiptsTests[i].root = types.DeriveSha(receipts, trie.NewStackTrie(nil))
}
}
func TestReceiptList69(t *testing.T) {
for i, test := range receiptsTests {
// encode receipts from types.ReceiptForStorage object.
canonDB, _ := rlp.EncodeToBytes(test.input)
// encode block body from types object.
blockBody := types.Body{Transactions: test.txs}
canonBody, _ := rlp.EncodeToBytes(blockBody)
// convert from storage encoding to network encoding
network, err := blockReceiptsToNetwork69(canonDB, canonBody)
if err != nil {
t.Fatalf("test[%d]: blockReceiptsToNetwork69 error: %v", i, err)
}
// parse as Receipts response list from network encoding
var rl ReceiptList69
if err := rlp.DecodeBytes(network, &rl); err != nil {
t.Fatalf("test[%d]: can't decode network receipts: %v", i, err)
}
rlStorageEnc := rl.EncodeForStorage()
if !bytes.Equal(rlStorageEnc, canonDB) {
t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB)
}
rlNetworkEnc, _ := rlp.EncodeToBytes(&rl)
if !bytes.Equal(rlNetworkEnc, network) {
t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network)
}
// compute root hash from ReceiptList69 and compare.
responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil))
if responseHash != test.root {
t.Fatalf("test[%d]: wrong root hash from ReceiptList69\nhave: %v\nwant: %v", i, responseHash, test.root)
}
}
}
func TestReceiptList68(t *testing.T) {
for i, test := range receiptsTests {
// encode receipts from types.ReceiptForStorage object.
canonDB, _ := rlp.EncodeToBytes(test.input)
// encode block body from types object.
blockBody := types.Body{Transactions: test.txs}
canonBody, _ := rlp.EncodeToBytes(blockBody)
// convert from storage encoding to network encoding
network, err := blockReceiptsToNetwork68(canonDB, canonBody)
if err != nil {
t.Fatalf("test[%d]: blockReceiptsToNetwork68 error: %v", i, err)
}
// parse as Receipts response list from network encoding
var rl ReceiptList68
if err := rlp.DecodeBytes(network, &rl); err != nil {
t.Fatalf("test[%d]: can't decode network receipts: %v", i, err)
}
rlStorageEnc := rl.EncodeForStorage()
if !bytes.Equal(rlStorageEnc, canonDB) {
t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB)
}
rlNetworkEnc, _ := rlp.EncodeToBytes(&rl)
if !bytes.Equal(rlNetworkEnc, network) {
t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network)
}
// compute root hash from ReceiptList68 and compare.
responseHash := types.DeriveSha(&rl, trie.NewStackTrie(nil))
if responseHash != test.root {
t.Fatalf("test[%d]: wrong root hash from ReceiptList68\nhave: %v\nwant: %v", i, responseHash, test.root)
}
}
}

26
eth/protocols/eth/tracker.go
View file

@ -1,26 +0,0 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"time"
"github.com/ethereum/go-ethereum/p2p/tracker"
)
// requestTracker is a singleton tracker for eth/66 and newer request times.
var requestTracker = tracker.New(ProtocolName, 5*time.Minute)

44
eth/protocols/snap/handler.go
View file

@ -23,6 +23,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
@ -31,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/enr"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/triedb/database"
)
const (
@ -279,7 +282,16 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac
if err != nil {
return nil, nil
}
it, err := chain.Snapshots().AccountIterator(req.Root, req.Origin)
// Temporary solution: using the snapshot interface for both cases.
// This can be removed once the hash scheme is deprecated.
var it snapshot.AccountIterator
if chain.TrieDB().Scheme() == rawdb.HashScheme {
// The snapshot is assumed to be available in hash mode if
// the SNAP protocol is enabled.
it, err = chain.Snapshots().AccountIterator(req.Root, req.Origin)
} else {
it, err = chain.TrieDB().AccountIterator(req.Root, req.Origin)
}
if err != nil {
return nil, nil
}
@ -359,7 +371,19 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP
limit, req.Limit = common.BytesToHash(req.Limit), nil
}
// Retrieve the requested state and bail out if non existent
it, err := chain.Snapshots().StorageIterator(req.Root, account, origin)
var (
err error
it snapshot.StorageIterator
)
// Temporary solution: using the snapshot interface for both cases.
// This can be removed once the hash scheme is deprecated.
if chain.TrieDB().Scheme() == rawdb.HashScheme {
// The snapshot is assumed to be available in hash mode if
// the SNAP protocol is enabled.
it, err = chain.Snapshots().StorageIterator(req.Root, account, origin)
} else {
it, err = chain.TrieDB().StorageIterator(req.Root, account, origin)
}
if err != nil {
return nil, nil
}
@ -479,8 +503,15 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
// We don't have the requested state available, bail out
return nil, nil
}
// The 'snap' might be nil, in which case we cannot serve storage slots.
snap := chain.Snapshots().Snapshot(req.Root)
// The 'reader' might be nil, in which case we cannot serve storage slots
// via snapshot.
var reader database.StateReader
if chain.Snapshots() != nil {
reader = chain.Snapshots().Snapshot(req.Root)
}
if reader == nil {
reader, _ = triedb.StateReader(req.Root)
}
// Retrieve trie nodes until the packet size limit is reached
var (
nodes [][]byte
@ -505,8 +536,9 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
default:
var stRoot common.Hash
// Storage slots requested, open the storage trie and retrieve from there
if snap == nil {
if reader == nil {
// We don't have the requested state snapshotted yet (or it is stale),
// but can look up the account via the trie instead.
account, err := accTrie.GetAccountByHash(common.BytesToHash(pathset[0]))
@ -516,7 +548,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s
}
stRoot = account.Root
} else {
account, err := snap.Account(common.BytesToHash(pathset[0]))
account, err := reader.Account(common.BytesToHash(pathset[0]))
loads++ // always account database reads, even for failures
if err != nil || account == nil {
break

2
eth/protocols/snap/sync_test.go
View file

@ -1962,5 +1962,5 @@ func newDbConfig(scheme string) *triedb.Config {
if scheme == rawdb.HashScheme {
return &triedb.Config{}
}
return &triedb.Config{PathDB: pathdb.Defaults}
return &triedb.Config{PathDB: &pathdb.Config{SnapshotNoBuild: true}}
}

2
tests/block_test_util.go
View file

@ -187,10 +187,12 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t
}
// Cross-check the snapshot-to-hash against the trie hash
if snapshotter {
if chain.Snapshots() != nil {
if err := chain.Snapshots().Verify(chain.CurrentBlock().Root); err != nil {
return err
}
}
}
return t.validateImportedHeaders(chain, validBlocks)
}

6
trie/database_test.go
View file

@ -25,7 +25,7 @@ import (
"github.com/ethereum/go-ethereum/triedb/database"
)
// testReader implements database.Reader interface, providing function to
// testReader implements database.NodeReader interface, providing function to
// access trie nodes.
type testReader struct {
db ethdb.Database
@ -33,7 +33,7 @@ type testReader struct {
nodes []*trienode.MergedNodeSet // sorted from new to old
}
// Node implements database.Reader interface, retrieving trie node with
// Node implements database.NodeReader interface, retrieving trie node with
// all available cached layers.
func (r *testReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
// Check the node presence with the cached layer, from latest to oldest.
@ -54,7 +54,7 @@ func (r *testReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
return rawdb.ReadTrieNode(r.db, owner, path, hash, r.scheme), nil
}
// testDb implements database.Database interface, using for testing purpose.
// testDb implements database.NodeDatabase interface, using for testing purpose.
type testDb struct {
disk ethdb.Database
root common.Hash

60
trie/secure_trie.go
View file

@ -19,6 +19,7 @@ package trie
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/triedb/database"
@ -63,8 +64,7 @@ type StateTrie struct {
trie Trie
db database.NodeDatabase
preimages preimageStore
hashKeyBuf [common.HashLength]byte
secKeyCache map[string][]byte
secKeyCache map[common.Hash][]byte
secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch
}
@ -97,7 +97,7 @@ func NewStateTrie(id *ID, db database.NodeDatabase) (*StateTrie, error) {
// This function will omit any encountered error but just
// print out an error message.
func (t *StateTrie) MustGet(key []byte) []byte {
return t.trie.MustGet(t.hashKey(key))
return t.trie.MustGet(crypto.Keccak256(key))
}
// GetStorage attempts to retrieve a storage slot with provided account address
@ -105,7 +105,7 @@ func (t *StateTrie) MustGet(key []byte) []byte {
// If the specified storage slot is not in the trie, nil will be returned.
// If a trie node is not found in the database, a MissingNodeError is returned.
func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) {
enc, err := t.trie.Get(t.hashKey(key))
enc, err := t.trie.Get(crypto.Keccak256(key))
if err != nil || len(enc) == 0 {
return nil, err
}
@ -117,7 +117,7 @@ func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) {
// If the specified account is not in the trie, nil will be returned.
// If a trie node is not found in the database, a MissingNodeError is returned.
func (t *StateTrie) GetAccount(address common.Address) (*types.StateAccount, error) {
res, err := t.trie.Get(t.hashKey(address.Bytes()))
res, err := t.trie.Get(crypto.Keccak256(address.Bytes()))
if res == nil || err != nil {
return nil, err
}
@ -157,9 +157,9 @@ func (t *StateTrie) GetNode(path []byte) ([]byte, int, error) {
// This function will omit any encountered error but just print out an
// error message.
func (t *StateTrie) MustUpdate(key, value []byte) {
hk := t.hashKey(key)
hk := crypto.Keccak256(key)
t.trie.MustUpdate(hk, value)
t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
t.getSecKeyCache()[common.Hash(hk)] = common.CopyBytes(key)
}
// UpdateStorage associates key with value in the trie. Subsequent calls to
@ -171,19 +171,19 @@ func (t *StateTrie) MustUpdate(key, value []byte) {
//
// If a node is not found in the database, a MissingNodeError is returned.
func (t *StateTrie) UpdateStorage(_ common.Address, key, value []byte) error {
hk := t.hashKey(key)
hk := crypto.Keccak256(key)
v, _ := rlp.EncodeToBytes(value)
err := t.trie.Update(hk, v)
if err != nil {
return err
}
t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
t.getSecKeyCache()[common.Hash(hk)] = common.CopyBytes(key)
return nil
}
// UpdateAccount will abstract the write of an account to the secure trie.
func (t *StateTrie) UpdateAccount(address common.Address, acc *types.StateAccount, _ int) error {
hk := t.hashKey(address.Bytes())
hk := crypto.Keccak256(address.Bytes())
data, err := rlp.EncodeToBytes(acc)
if err != nil {
return err
@ -191,7 +191,7 @@ func (t *StateTrie) UpdateAccount(address common.Address, acc *types.StateAccoun
if err := t.trie.Update(hk, data); err != nil {
return err
}
t.getSecKeyCache()[string(hk)] = address.Bytes()
t.getSecKeyCache()[common.Hash(hk)] = address.Bytes()
return nil
}
@ -202,8 +202,8 @@ func (t *StateTrie) UpdateContractCode(_ common.Address, _ common.Hash, _ []byte
// MustDelete removes any existing value for key from the trie. This function
// will omit any encountered error but just print out an error message.
func (t *StateTrie) MustDelete(key []byte) {
hk := t.hashKey(key)
delete(t.getSecKeyCache(), string(hk))
hk := crypto.Keccak256(key)
delete(t.getSecKeyCache(), common.Hash(hk))
t.trie.MustDelete(hk)
}
@ -211,22 +211,22 @@ func (t *StateTrie) MustDelete(key []byte) {
// If the specified trie node is not in the trie, nothing will be changed.
// If a node is not found in the database, a MissingNodeError is returned.
func (t *StateTrie) DeleteStorage(_ common.Address, key []byte) error {
hk := t.hashKey(key)
delete(t.getSecKeyCache(), string(hk))
hk := crypto.Keccak256(key)
delete(t.getSecKeyCache(), common.Hash(hk))
return t.trie.Delete(hk)
}
// DeleteAccount abstracts an account deletion from the trie.
func (t *StateTrie) DeleteAccount(address common.Address) error {
hk := t.hashKey(address.Bytes())
delete(t.getSecKeyCache(), string(hk))
hk := crypto.Keccak256(address.Bytes())
delete(t.getSecKeyCache(), common.Hash(hk))
return t.trie.Delete(hk)
}
// GetKey returns the sha3 preimage of a hashed key that was
// previously used to store a value.
func (t *StateTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
if key, ok := t.getSecKeyCache()[common.BytesToHash(shaKey)]; ok {
return key
}
if t.preimages == nil {
@ -251,13 +251,9 @@ func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
// Write all the pre-images to the actual disk database
if len(t.getSecKeyCache()) > 0 {
if t.preimages != nil {
preimages := make(map[common.Hash][]byte, len(t.secKeyCache))
for hk, key := range t.secKeyCache {
preimages[common.BytesToHash([]byte(hk))] = key
t.preimages.InsertPreimage(t.secKeyCache)
}
t.preimages.InsertPreimage(preimages)
}
t.secKeyCache = make(map[string][]byte)
t.secKeyCache = make(map[common.Hash][]byte)
}
// Commit the trie and return its modified nodeset.
return t.trie.Commit(collectLeaf)
@ -291,25 +287,13 @@ func (t *StateTrie) MustNodeIterator(start []byte) NodeIterator {
return t.trie.MustNodeIterator(start)
}
// hashKey returns the hash of key as an ephemeral buffer.
// The caller must not hold onto the return value because it will become
// invalid on the next call to hashKey or secKey.
func (t *StateTrie) hashKey(key []byte) []byte {
h := newHasher(false)
h.sha.Reset()
h.sha.Write(key)
h.sha.Read(t.hashKeyBuf[:])
returnHasherToPool(h)
return t.hashKeyBuf[:]
}
// getSecKeyCache returns the current secure key cache, creating a new one if
// ownership changed (i.e. the current secure trie is a copy of another owning
// the actual cache).
func (t *StateTrie) getSecKeyCache() map[string][]byte {
func (t *StateTrie) getSecKeyCache() map[common.Hash][]byte {
if t != t.secKeyCacheOwner {
t.secKeyCacheOwner = t
t.secKeyCache = make(map[string][]byte)
t.secKeyCache = make(map[common.Hash][]byte)
}
return t.secKeyCache
}

20
triedb/database.go
View file

@ -312,6 +312,26 @@ func (db *Database) Journal(root common.Hash) error {
return pdb.Journal(root)
}
// AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash.
func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (pathdb.AccountIterator, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return nil, errors.New("not supported")
}
return pdb.AccountIterator(root, seek)
}
// StorageIterator creates a new storage iterator for the specified root hash and
// account. The iterator will be move to the specific start position.
func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (pathdb.StorageIterator, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return nil, errors.New("not supported")
}
return pdb.StorageIterator(root, account, seek)
}
// IsVerkle returns the indicator if the database is holding a verkle tree.
func (db *Database) IsVerkle() bool {
return db.config.IsVerkle

10
triedb/pathdb/buffer.go
View file

@ -124,7 +124,7 @@ func (b *buffer) size() uint64 {
// flush persists the in-memory dirty trie node into the disk if the configured
// memory threshold is reached. Note, all data must be written atomically.
func (b *buffer) flush(db ethdb.KeyValueStore, freezer ethdb.AncientWriter, nodesCache *fastcache.Cache, id uint64) error {
func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64) error {
// Ensure the target state id is aligned with the internal counter.
head := rawdb.ReadPersistentStateID(db)
if head+b.layers != id {
@ -133,7 +133,7 @@ func (b *buffer) flush(db ethdb.KeyValueStore, freezer ethdb.AncientWriter, node
// Terminate the state snapshot generation if it's active
var (
start = time.Now()
batch = db.NewBatchWithSize(b.nodes.dbsize() * 11 / 10) // extra 10% for potential pebble internal stuff
batch = db.NewBatchWithSize((b.nodes.dbsize() + b.states.dbsize()) * 11 / 10) // extra 10% for potential pebble internal stuff
)
// Explicitly sync the state freezer to ensure all written data is persisted to disk
// before updating the key-value store.
@ -146,7 +146,9 @@ func (b *buffer) flush(db ethdb.KeyValueStore, freezer ethdb.AncientWriter, node
}
}
nodes := b.nodes.write(batch, nodesCache)
accounts, slots := b.states.write(batch, progress, statesCache)
rawdb.WritePersistentStateID(batch, id)
rawdb.WriteSnapshotRoot(batch, root)
// Flush all mutations in a single batch
size := batch.ValueSize()
@ -155,8 +157,10 @@ func (b *buffer) flush(db ethdb.KeyValueStore, freezer ethdb.AncientWriter, node
}
commitBytesMeter.Mark(int64(size))
commitNodesMeter.Mark(int64(nodes))
commitAccountsMeter.Mark(int64(accounts))
commitStoragesMeter.Mark(int64(slots))
commitTimeTimer.UpdateSince(start)
b.reset()
log.Debug("Persisted buffer content", "nodes", nodes, "bytes", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
log.Debug("Persisted buffer content", "nodes", nodes, "accounts", accounts, "slots", slots, "bytes", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}

246
triedb/pathdb/context.go Normal file
View file

@ -0,0 +1,246 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pathdb
import (
"bytes"
"encoding/binary"
"errors"
"math"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log"
)
const (
snapAccount = "account" // Identifier of account snapshot generation
snapStorage = "storage" // Identifier of storage snapshot generation
)
// generatorStats is a collection of statistics gathered by the snapshot generator
// for logging purposes. This data structure is used throughout the entire
// lifecycle of the snapshot generation process and is shared across multiple
// generation cycles.
type generatorStats struct {
origin uint64 // Origin prefix where generation started
start time.Time // Timestamp when generation started
accounts uint64 // Number of accounts indexed(generated or recovered)
slots uint64 // Number of storage slots indexed(generated or recovered)
dangling uint64 // Number of dangling storage slots
storage common.StorageSize // Total account and storage slot size(generation or recovery)
}
// log creates a contextual log with the given message and the context pulled
// from the internally maintained statistics.
func (gs *generatorStats) log(msg string, root common.Hash, marker []byte) {
var ctx []interface{}
if root != (common.Hash{}) {
ctx = append(ctx, []interface{}{"root", root}...)
}
// Figure out whether we're after or within an account
switch len(marker) {
case common.HashLength:
ctx = append(ctx, []interface{}{"at", common.BytesToHash(marker)}...)
case 2 * common.HashLength:
ctx = append(ctx, []interface{}{
"in", common.BytesToHash(marker[:common.HashLength]),
"at", common.BytesToHash(marker[common.HashLength:]),
}...)
}
// Add the usual measurements
ctx = append(ctx, []interface{}{
"accounts", gs.accounts,
"slots", gs.slots,
"storage", gs.storage,
"dangling", gs.dangling,
"elapsed", common.PrettyDuration(time.Since(gs.start)),
}...)
// Calculate the estimated indexing time based on current stats
if len(marker) > 0 {
if done := binary.BigEndian.Uint64(marker[:8]) - gs.origin; done > 0 {
left := math.MaxUint64 - binary.BigEndian.Uint64(marker[:8])
speed := done/uint64(time.Since(gs.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
ctx = append(ctx, []interface{}{
"eta", common.PrettyDuration(time.Duration(left/speed) * time.Millisecond),
}...)
}
}
log.Info(msg, ctx...)
}
// generatorContext holds several global fields that are used throughout the
// current generation cycle. It must be recreated if the generation cycle is
// restarted.
type generatorContext struct {
root common.Hash // State root of the generation target
account *holdableIterator // Iterator of account snapshot data
storage *holdableIterator // Iterator of storage snapshot data
db ethdb.KeyValueStore // Key-value store containing the snapshot data
batch ethdb.Batch // Database batch for writing data atomically
logged time.Time // The timestamp when last generation progress was displayed
}
// newGeneratorContext initializes the context for generation.
func newGeneratorContext(root common.Hash, marker []byte, db ethdb.KeyValueStore) *generatorContext {
ctx := &generatorContext{
root: root,
db: db,
batch: db.NewBatch(),
logged: time.Now(),
}
accMarker, storageMarker := splitMarker(marker)
ctx.openIterator(snapAccount, accMarker)
ctx.openIterator(snapStorage, storageMarker)
return ctx
}
// openIterator constructs global account and storage snapshot iterators
// at the interrupted position. These iterators should be reopened from time
// to time to avoid blocking leveldb compaction for a long time.
func (ctx *generatorContext) openIterator(kind string, start []byte) {
if kind == snapAccount {
iter := ctx.db.NewIterator(rawdb.SnapshotAccountPrefix, start)
ctx.account = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+common.HashLength))
return
}
iter := ctx.db.NewIterator(rawdb.SnapshotStoragePrefix, start)
ctx.storage = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+2*common.HashLength))
}
// reopenIterator releases the specified snapshot iterator and re-open it
// in the next position. It's aimed for not blocking leveldb compaction.
func (ctx *generatorContext) reopenIterator(kind string) {
// Shift iterator one more step, so that we can reopen
// the iterator at the right position.
var iter = ctx.account
if kind == snapStorage {
iter = ctx.storage
}
hasNext := iter.Next()
if !hasNext {
// Iterator exhausted, release forever and create an already exhausted virtual iterator
iter.Release()
if kind == snapAccount {
ctx.account = newHoldableIterator(memorydb.New().NewIterator(nil, nil))
return
}
ctx.storage = newHoldableIterator(memorydb.New().NewIterator(nil, nil))
return
}
next := iter.Key()
iter.Release()
ctx.openIterator(kind, next[1:])
}
// close releases all the held resources.
func (ctx *generatorContext) close() {
ctx.account.Release()
ctx.storage.Release()
}
// iterator returns the corresponding iterator specified by the kind.
func (ctx *generatorContext) iterator(kind string) *holdableIterator {
if kind == snapAccount {
return ctx.account
}
return ctx.storage
}
// removeStorageBefore deletes all storage entries which are located before
// the specified account. When the iterator touches the storage entry which
// is located in or outside the given account, it stops and holds the current
// iterated element locally.
func (ctx *generatorContext) removeStorageBefore(account common.Hash) uint64 {
var (
count uint64
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
key := iter.Key()
if bytes.Compare(key[1:1+common.HashLength], account.Bytes()) >= 0 {
iter.Hold()
break
}
count++
ctx.batch.Delete(key)
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
}
}
storageCleanCounter.Inc(time.Since(start).Nanoseconds())
return count
}
// removeStorageAt deletes all storage entries which are located in the specified
// account. When the iterator touches the storage entry which is outside the given
// account, it stops and holds the current iterated element locally. An error will
// be returned if the initial position of iterator is not in the given account.
func (ctx *generatorContext) removeStorageAt(account common.Hash) error {
var (
count int64
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
key := iter.Key()
cmp := bytes.Compare(key[1:1+common.HashLength], account.Bytes())
if cmp < 0 {
return errors.New("invalid iterator position")
}
if cmp > 0 {
iter.Hold()
break
}
count++
ctx.batch.Delete(key)
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
}
}
wipedStorageMeter.Mark(count)
storageCleanCounter.Inc(time.Since(start).Nanoseconds())
return nil
}
// removeRemainingStorage deletes all storage entries which are located after
// the current iterator position.
func (ctx *generatorContext) removeRemainingStorage() uint64 {
var (
count uint64
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
count++
ctx.batch.Delete(iter.Key())
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
}
}
danglingStorageMeter.Mark(int64(count))
storageCleanCounter.Inc(time.Since(start).Nanoseconds())
return count
}

138
triedb/pathdb/database.go
View file

@ -17,6 +17,7 @@
package pathdb
import (
"encoding/binary"
"errors"
"fmt"
"io"
@ -35,8 +36,11 @@ import (
)
const (
// defaultCleanSize is the default memory allowance of clean cache.
defaultCleanSize = 16 * 1024 * 1024
// defaultTrieCleanSize is the default memory allowance of clean trie cache.
defaultTrieCleanSize = 16 * 1024 * 1024
// defaultStateCleanSize is the default memory allowance of clean state cache.
defaultStateCleanSize = 16 * 1024 * 1024
// maxBufferSize is the maximum memory allowance of node buffer.
// Too large buffer will cause the system to pause for a long
@ -111,9 +115,11 @@ type layer interface {
// Config contains the settings for database.
type Config struct {
StateHistory uint64 // Number of recent blocks to maintain state history for
CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie nodes
StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data
WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer
ReadOnly bool // Flag whether the database is opened in read only mode.
ReadOnly bool // Flag whether the database is opened in read only mode
SnapshotNoBuild bool // Flag Whether the background generation is allowed
}
// sanitize checks the provided user configurations and changes anything that's
@ -133,7 +139,11 @@ func (c *Config) fields() []interface{} {
if c.ReadOnly {
list = append(list, "readonly", true)
}
list = append(list, "cache", common.StorageSize(c.CleanCacheSize))
if c.SnapshotNoBuild {
list = append(list, "snapshot", false)
}
list = append(list, "triecache", common.StorageSize(c.TrieCleanSize))
list = append(list, "statecache", common.StorageSize(c.StateCleanSize))
list = append(list, "buffer", common.StorageSize(c.WriteBufferSize))
list = append(list, "history", c.StateHistory)
return list
@ -142,7 +152,8 @@ func (c *Config) fields() []interface{} {
// Defaults contains default settings for Ethereum mainnet.
var Defaults = &Config{
StateHistory: params.FullImmutabilityThreshold,
CleanCacheSize: defaultCleanSize,
TrieCleanSize: defaultTrieCleanSize,
StateCleanSize: defaultStateCleanSize,
WriteBufferSize: defaultBufferSize,
}
@ -240,6 +251,13 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
log.Crit("Failed to disable database", "err", err) // impossible to happen
}
}
// Resolving the state snapshot generation progress from the database is
// mandatory. This ensures that uncovered flat states are not accessed,
// even if background generation is not allowed. If permitted, the generation
// might be scheduled.
if err := db.setStateGenerator(); err != nil {
log.Crit("Failed to setup the generator", "err", err)
}
fields := config.fields()
if db.isVerkle {
fields = append(fields, "verkle", true)
@ -297,6 +315,60 @@ func (db *Database) repairHistory() error {
return nil
}
// setStateGenerator loads the state generation progress marker and potentially
// resume the state generation if it's permitted.
func (db *Database) setStateGenerator() error {
// Load the state snapshot generation progress marker to prevent access
// to uncovered states.
generator, root, err := loadGenerator(db.diskdb, db.hasher)
if err != nil {
return err
}
if generator == nil {
// Initialize an empty generator to rebuild the state snapshot from scratch
generator = &journalGenerator{
Marker: []byte{},
}
}
// Short circuit if the whole state snapshot has already been fully generated.
// The generator will be left as nil in disk layer for representing the whole
// state snapshot is available for accessing.
if generator.Done {
return nil
}
var origin uint64
if len(generator.Marker) >= 8 {
origin = binary.BigEndian.Uint64(generator.Marker)
}
stats := &generatorStats{
origin: origin,
start: time.Now(),
accounts: generator.Accounts,
slots: generator.Slots,
storage: common.StorageSize(generator.Storage),
}
dl := db.tree.bottom()
// Disable the background snapshot building in these circumstances:
// - the database is opened in read only mode
// - the snapshot build is explicitly disabled
// - the database is opened in verkle tree mode
noBuild := db.readOnly || db.config.SnapshotNoBuild || db.isVerkle
// Construct the generator and link it to the disk layer, ensuring that the
// generation progress is resolved to prevent accessing uncovered states
// regardless of whether background state snapshot generation is allowed.
dl.setGenerator(newGenerator(db.diskdb, noBuild, generator.Marker, stats))
// Short circuit if the background generation is not permitted
if noBuild || db.waitSync {
return nil
}
stats.log("Starting snapshot generation", root, generator.Marker)
dl.generator.run(root)
return nil
}
// Update adds a new layer into the tree, if that can be linked to an existing
// old parent. It is disallowed to insert a disk layer (the origin of all). Apart
// from that this function will flatten the extra diff layers at bottom into disk
@ -359,8 +431,13 @@ func (db *Database) Disable() error {
}
db.waitSync = true
// Mark the disk layer as stale to prevent access to persistent state.
db.tree.bottom().markStale()
// Terminate the state generator if it's active and mark the disk layer
// as stale to prevent access to persistent state.
disk := db.tree.bottom()
if disk.generator != nil {
disk.generator.stop()
}
disk.markStale()
// Write the initial sync flag to persist it across restarts.
rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncRunning)
@ -390,6 +467,7 @@ func (db *Database) Enable(root common.Hash) error {
// reset the persistent state id back to zero.
batch := db.diskdb.NewBatch()
rawdb.DeleteTrieJournal(batch)
rawdb.DeleteSnapshotRoot(batch)
rawdb.WritePersistentStateID(batch, 0)
if err := batch.Write(); err != nil {
return err
@ -403,13 +481,13 @@ func (db *Database) Enable(root common.Hash) error {
return err
}
}
// Re-construct a new disk layer backed by persistent state
// with **empty clean cache and node buffer**.
db.tree.reset(newDiskLayer(root, 0, db, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0)))
// Re-enable the database as the final step.
db.waitSync = false
rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished)
// Re-construct a new disk layer backed by persistent state
// and schedule the state snapshot generation if it's permitted.
db.tree.reset(generateSnapshot(db, root, db.isVerkle || db.config.SnapshotNoBuild))
log.Info("Rebuilt trie database", "root", root)
return nil
}
@ -514,8 +592,12 @@ func (db *Database) Close() error {
// following mutations.
db.readOnly = true
// Release the memory held by clean cache.
db.tree.bottom().resetCache()
// Terminate the background generation if it's active
disk := db.tree.bottom()
if disk.generator != nil {
disk.generator.stop()
}
disk.resetCache() // release the memory held by clean cache
// Close the attached state history freezer.
if db.freezer == nil {
@ -580,14 +662,42 @@ func (db *Database) HistoryRange() (uint64, uint64, error) {
return historyRange(db.freezer)
}
// waitGeneration waits until the background generation is finished. It assumes
// that the generation is permitted; otherwise, it will block indefinitely.
func (db *Database) waitGeneration() {
gen := db.tree.bottom().generator
if gen == nil || gen.completed() {
return
}
<-gen.done
}
// AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash.
func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {
db.lock.RLock()
wait := db.waitSync
db.lock.RUnlock()
if wait {
return nil, errDatabaseWaitSync
}
if gen := db.tree.bottom().generator; gen != nil && !gen.completed() {
return nil, errNotConstructed
}
return newFastAccountIterator(db, root, seek)
}
// StorageIterator creates a new storage iterator for the specified root hash and
// account. The iterator will be moved to the specific start position.
func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) {
db.lock.RLock()
wait := db.waitSync
db.lock.RUnlock()
if wait {
return nil, errDatabaseWaitSync
}
if gen := db.tree.bottom().generator; gen != nil && !gen.completed() {
return nil, errNotConstructed
}
return newFastStorageIterator(db, root, account, seek)
}

3
triedb/pathdb/database_test.go
View file

@ -126,7 +126,8 @@ func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int) *te
disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
db = New(disk, &Config{
StateHistory: historyLimit,
CleanCacheSize: 256 * 1024,
TrieCleanSize: 256 * 1024,
StateCleanSize: 256 * 1024,
WriteBufferSize: 256 * 1024,
}, isVerkle)

185
triedb/pathdb/disklayer.go
View file

@ -17,9 +17,10 @@
package pathdb
import (
"errors"
"bytes"
"fmt"
"sync"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
@ -33,25 +34,39 @@ type diskLayer struct {
root common.Hash // Immutable, root hash to which this layer was made for
id uint64 // Immutable, corresponding state id
db *Database // Path-based trie database
// These two caches must be maintained separately, because the key
// for the root node of the storage trie (accountHash) is identical
// to the key for the account data.
nodes *fastcache.Cache // GC friendly memory cache of clean nodes
states *fastcache.Cache // GC friendly memory cache of clean states
buffer *buffer // Dirty buffer to aggregate writes of nodes and states
stale bool // Signals that the layer became stale (state progressed)
lock sync.RWMutex // Lock used to protect stale flag
lock sync.RWMutex // Lock used to protect stale flag and genMarker
// The generator is set if the state snapshot was not fully completed,
// regardless of whether the background generation is running or not.
generator *generator
}
// newDiskLayer creates a new disk layer based on the passing arguments.
func newDiskLayer(root common.Hash, id uint64, db *Database, nodes *fastcache.Cache, buffer *buffer) *diskLayer {
// Initialize a clean cache if the memory allowance is not zero
// or reuse the provided cache if it is not nil (inherited from
func newDiskLayer(root common.Hash, id uint64, db *Database, nodes *fastcache.Cache, states *fastcache.Cache, buffer *buffer) *diskLayer {
// Initialize the clean caches if the memory allowance is not zero
// or reuse the provided caches if they are not nil (inherited from
// the original disk layer).
if nodes == nil && db.config.CleanCacheSize != 0 {
nodes = fastcache.New(db.config.CleanCacheSize)
if nodes == nil && db.config.TrieCleanSize != 0 {
nodes = fastcache.New(db.config.TrieCleanSize)
}
if states == nil && db.config.StateCleanSize != 0 {
states = fastcache.New(db.config.StateCleanSize)
}
return &diskLayer{
root: root,
id: id,
db: db,
nodes: nodes,
states: states,
buffer: buffer,
}
}
@ -72,6 +87,13 @@ func (dl *diskLayer) parentLayer() layer {
return nil
}
// setGenerator links the given generator to disk layer, representing the
// associated state snapshot is not fully completed yet and the generation
// is potentially running in the background.
func (dl *diskLayer) setGenerator(generator *generator) {
dl.generator = generator
}
// isStale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diskLayer) isStale() bool {
@ -168,8 +190,41 @@ func (dl *diskLayer) account(hash common.Hash, depth int) ([]byte, error) {
}
dirtyStateMissMeter.Mark(1)
// TODO(rjl493456442) support persistent state retrieval
return nil, errors.New("not supported")
// If the layer is being generated, ensure the requested account has
// already been covered by the generator.
marker := dl.genMarker()
if marker != nil && bytes.Compare(hash.Bytes(), marker) > 0 {
return nil, errNotCoveredYet
}
// Try to retrieve the account from the memory cache
if dl.states != nil {
if blob, found := dl.states.HasGet(nil, hash[:]); found {
cleanStateHitMeter.Mark(1)
cleanStateReadMeter.Mark(int64(len(blob)))
if len(blob) == 0 {
stateAccountInexMeter.Mark(1)
} else {
stateAccountExistMeter.Mark(1)
}
return blob, nil
}
cleanStateMissMeter.Mark(1)
}
// Try to retrieve the account from the disk.
blob = rawdb.ReadAccountSnapshot(dl.db.diskdb, hash)
if dl.states != nil {
dl.states.Set(hash[:], blob)
cleanStateWriteMeter.Mark(int64(len(blob)))
}
if len(blob) == 0 {
stateAccountInexMeter.Mark(1)
stateAccountInexDiskMeter.Mark(1)
} else {
stateAccountExistMeter.Mark(1)
stateAccountExistDiskMeter.Mark(1)
}
return blob, nil
}
// storage directly retrieves the storage data associated with a particular hash,
@ -203,8 +258,42 @@ func (dl *diskLayer) storage(accountHash, storageHash common.Hash, depth int) ([
}
dirtyStateMissMeter.Mark(1)
// TODO(rjl493456442) support persistent state retrieval
return nil, errors.New("not supported")
// If the layer is being generated, ensure the requested storage slot
// has already been covered by the generator.
key := append(accountHash[:], storageHash[:]...)
marker := dl.genMarker()
if marker != nil && bytes.Compare(key, marker) > 0 {
return nil, errNotCoveredYet
}
// Try to retrieve the storage slot from the memory cache
if dl.states != nil {
if blob, found := dl.states.HasGet(nil, key); found {
cleanStateHitMeter.Mark(1)
cleanStateReadMeter.Mark(int64(len(blob)))
if len(blob) == 0 {
stateStorageInexMeter.Mark(1)
} else {
stateStorageExistMeter.Mark(1)
}
return blob, nil
}
cleanStateMissMeter.Mark(1)
}
// Try to retrieve the account from the disk
blob := rawdb.ReadStorageSnapshot(dl.db.diskdb, accountHash, storageHash)
if dl.states != nil {
dl.states.Set(key, blob)
cleanStateWriteMeter.Mark(int64(len(blob)))
}
if len(blob) == 0 {
stateStorageInexMeter.Mark(1)
stateStorageInexDiskMeter.Mark(1)
} else {
stateStorageExistMeter.Mark(1)
stateStorageExistDiskMeter.Mark(1)
}
return blob, nil
}
// update implements the layer interface, returning a new diff layer on top
@ -267,13 +356,39 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// Merge the trie nodes and flat states of the bottom-most diff layer into the
// buffer as the combined layer.
combined := dl.buffer.commit(bottom.nodes, bottom.states.stateSet)
// Terminate the background state snapshot generation before mutating the
// persistent state.
if combined.full() || force {
if err := combined.flush(dl.db.diskdb, dl.db.freezer, dl.nodes, bottom.stateID()); err != nil {
// Terminate the background state snapshot generator before flushing
// to prevent data race.
var progress []byte
if dl.generator != nil {
dl.generator.stop()
progress = dl.generator.progressMarker()
// If the snapshot has been fully generated, unset the generator
if progress == nil {
dl.setGenerator(nil)
} else {
log.Info("Paused snapshot generation")
}
}
// Flush the content in combined buffer. Any state data after the progress
// marker will be ignored, as the generator will pick it up later.
if err := combined.flush(bottom.root, dl.db.diskdb, dl.db.freezer, progress, dl.nodes, dl.states, bottom.stateID()); err != nil {
return nil, err
}
// Resume the background generation if it's not completed yet
if progress != nil {
dl.generator.run(bottom.root)
}
}
// Link the generator if snapshot is not yet completed
ndl := newDiskLayer(bottom.root, bottom.stateID(), dl.db, dl.nodes, dl.states, combined)
if dl.generator != nil {
ndl.setGenerator(dl.generator)
}
ndl := newDiskLayer(bottom.root, bottom.stateID(), dl.db, dl.nodes, combined)
// To remove outdated history objects from the end, we set the 'tail' parameter
// to 'oldest-1' due to the offset between the freezer index and the history ID.
if overflow {
@ -288,6 +403,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// revert applies the given state history and return a reverted disk layer.
func (dl *diskLayer) revert(h *history) (*diskLayer, error) {
start := time.Now()
if h.meta.root != dl.rootHash() {
return nil, errUnexpectedHistory
}
@ -321,15 +437,40 @@ func (dl *diskLayer) revert(h *history) (*diskLayer, error) {
if err != nil {
return nil, err
}
} else {
ndl := newDiskLayer(h.meta.parent, dl.id-1, dl.db, dl.nodes, dl.states, dl.buffer)
// Link the generator if it exists
if dl.generator != nil {
ndl.setGenerator(dl.generator)
}
log.Debug("Reverted data in write buffer", "oldroot", h.meta.root, "newroot", h.meta.parent, "elapsed", common.PrettyDuration(time.Since(start)))
return ndl, nil
}
// Terminate the generation before writing any data into database
var progress []byte
if dl.generator != nil {
dl.generator.stop()
progress = dl.generator.progressMarker()
}
batch := dl.db.diskdb.NewBatch()
writeNodes(batch, nodes, dl.nodes)
// Provide the original values of modified accounts and storages for revert
writeStates(batch, progress, accounts, storages, dl.states)
rawdb.WritePersistentStateID(batch, dl.id-1)
rawdb.WriteSnapshotRoot(batch, h.meta.parent)
if err := batch.Write(); err != nil {
log.Crit("Failed to write states", "err", err)
}
// Link the generator and resume generation if the snapshot is not yet
// fully completed.
ndl := newDiskLayer(h.meta.parent, dl.id-1, dl.db, dl.nodes, dl.states, dl.buffer)
if dl.generator != nil && !dl.generator.completed() {
ndl.generator = dl.generator
ndl.generator.run(h.meta.parent)
}
return newDiskLayer(h.meta.parent, dl.id-1, dl.db, dl.nodes, dl.buffer), nil
log.Debug("Reverted data in persistent state", "oldroot", h.meta.root, "newroot", h.meta.parent, "elapsed", common.PrettyDuration(time.Since(start)))
return ndl, nil
}
// size returns the approximate size of cached nodes in the disk layer.
@ -355,4 +496,16 @@ func (dl *diskLayer) resetCache() {
if dl.nodes != nil {
dl.nodes.Reset()
}
if dl.states != nil {
dl.states.Reset()
}
}
// genMarker returns the current state snapshot generation progress marker. If
// the state snapshot has already been fully generated, nil is returned.
func (dl *diskLayer) genMarker() []byte {
if dl.generator == nil {
return nil
}
return dl.generator.progressMarker()
}

9
triedb/pathdb/errors.go
View file

@ -39,4 +39,13 @@ var (
// errStateUnrecoverable is returned if state is required to be reverted to
// a destination without associated state history available.
errStateUnrecoverable = errors.New("state is unrecoverable")
// errNotCoveredYet is returned from data accessors if the underlying snapshot
// is being generated currently and the requested data item is not yet in the
// range of accounts covered.
errNotCoveredYet = errors.New("not covered yet")
// errNotConstructed is returned if the callers want to iterate the snapshot
// while the generation is not finished yet.
errNotConstructed = errors.New("snapshot is not constructed")
)

68
triedb/pathdb/flush.go
View file

@ -17,6 +17,8 @@
package pathdb
import (
"bytes"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
@ -63,3 +65,69 @@ func writeNodes(batch ethdb.Batch, nodes map[common.Hash]map[string]*trienode.No
}
return total
}
// writeStates flushes state mutations into the provided database batch as a whole.
//
// This function assumes the background generator is already terminated and states
// before the supplied marker has been correctly generated.
//
// TODO(rjl493456442) do we really need this generation marker? The state updates
// after the marker can also be written and will be fixed by generator later if
// it's outdated.
func writeStates(batch ethdb.Batch, genMarker []byte, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte, clean *fastcache.Cache) (int, int) {
var (
accounts int
slots int
)
for addrHash, blob := range accountData {
// Skip any account not yet covered by the snapshot. The account
// at the generation marker position (addrHash == genMarker[:common.HashLength])
// should still be updated, as it would be skipped in the next
// generation cycle.
if genMarker != nil && bytes.Compare(addrHash[:], genMarker) > 0 {
continue
}
accounts += 1
if len(blob) == 0 {
rawdb.DeleteAccountSnapshot(batch, addrHash)
if clean != nil {
clean.Set(addrHash[:], nil)
}
} else {
rawdb.WriteAccountSnapshot(batch, addrHash, blob)
if clean != nil {
clean.Set(addrHash[:], blob)
}
}
}
for addrHash, storages := range storageData {
// Skip any account not covered yet by the snapshot
if genMarker != nil && bytes.Compare(addrHash[:], genMarker) > 0 {
continue
}
midAccount := genMarker != nil && bytes.Equal(addrHash[:], genMarker[:common.HashLength])
for storageHash, blob := range storages {
// Skip any storage slot not yet covered by the snapshot. The storage slot
// at the generation marker position (addrHash == genMarker[:common.HashLength]
// and storageHash == genMarker[common.HashLength:]) should still be updated,
// as it would be skipped in the next generation cycle.
if midAccount && bytes.Compare(storageHash[:], genMarker[common.HashLength:]) > 0 {
continue
}
slots += 1
if len(blob) == 0 {
rawdb.DeleteStorageSnapshot(batch, addrHash, storageHash)
if clean != nil {
clean.Set(append(addrHash[:], storageHash[:]...), nil)
}
} else {
rawdb.WriteStorageSnapshot(batch, addrHash, storageHash, blob)
if clean != nil {
clean.Set(append(addrHash[:], storageHash[:]...), blob)
}
}
}
}
return accounts, slots
}

856
triedb/pathdb/generate.go Normal file
View file

@ -0,0 +1,856 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pathdb
import (
"bytes"
"errors"
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/triedb/database"
)
var (
// accountCheckRange is the upper limit of the number of accounts involved in
// each range check. This is a value estimated based on experience. If this
// range is too large, the failure rate of range proof will increase. Otherwise,
// if the range is too small, the efficiency of the state recovery will decrease.
accountCheckRange = 128
// storageCheckRange is the upper limit of the number of storage slots involved
// in each range check. This is a value estimated based on experience. If this
// range is too large, the failure rate of range proof will increase. Otherwise,
// if the range is too small, the efficiency of the state recovery will decrease.
storageCheckRange = 1024
// errMissingTrie is returned if the target trie is missing while the generation
// is running. In this case the generation is aborted and wait the new signal.
errMissingTrie = errors.New("missing trie")
)
// diskReader is a wrapper of key-value store and implements database.NodeReader,
// providing a function for accessing persistent trie nodes in the disk
type diskReader struct{ db ethdb.KeyValueStore }
// Node retrieves the trie node blob with the provided trie identifier,
// node path and the corresponding node hash. No error will be returned
// if the node is not found.
func (r *diskReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
if owner == (common.Hash{}) {
return rawdb.ReadAccountTrieNode(r.db, path), nil
}
return rawdb.ReadStorageTrieNode(r.db, owner, path), nil
}
// diskStore is a wrapper of key-value store and implements database.NodeDatabase.
// It's meant to be used for generating state snapshot from the trie data.
type diskStore struct {
db ethdb.KeyValueStore
}
// NodeReader returns a node reader associated with the specific state.
// An error will be returned if the specified state is not available.
func (s *diskStore) NodeReader(stateRoot common.Hash) (database.NodeReader, error) {
root := types.EmptyRootHash
if blob := rawdb.ReadAccountTrieNode(s.db, nil); len(blob) > 0 {
root = crypto.Keccak256Hash(blob)
}
if root != stateRoot {
return nil, fmt.Errorf("state %x is not available", stateRoot)
}
return &diskReader{s.db}, nil
}
// Generator is the struct for initial state snapshot generation. It is not thread-safe;
// the caller must manage concurrency issues themselves.
type generator struct {
noBuild bool // Flag indicating whether snapshot generation is permitted
running bool // Flag indicating whether the background generation is running
db ethdb.KeyValueStore // Key-value store containing the snapshot data
stats *generatorStats // Generation statistics used throughout the entire life cycle
abort chan chan struct{} // Notification channel to abort generating the snapshot in this layer
done chan struct{} // Notification channel when generation is done
progress []byte // Progress marker of the state generation, nil means it's completed
lock sync.RWMutex // Lock which protects the progress, only generator can mutate the progress
}
// newGenerator constructs the state snapshot generator.
//
// noBuild will be true if the background snapshot generation is not allowed,
// usually used in read-only mode.
//
// progress indicates the starting position for resuming snapshot generation.
// It must be provided even if generation is not allowed; otherwise, uncovered
// states may be exposed for serving.
func newGenerator(db ethdb.KeyValueStore, noBuild bool, progress []byte, stats *generatorStats) *generator {
if stats == nil {
stats = &generatorStats{start: time.Now()}
}
return &generator{
noBuild: noBuild,
progress: progress,
db: db,
stats: stats,
abort: make(chan chan struct{}),
done: make(chan struct{}),
}
}
// run starts the state snapshot generation in the background.
func (g *generator) run(root common.Hash) {
if g.noBuild {
log.Warn("Snapshot generation is not permitted")
return
}
if g.running {
g.stop()
log.Warn("Paused the leftover generation cycle")
}
g.running = true
go g.generate(newGeneratorContext(root, g.progress, g.db))
}
// stop terminates the background generation if it's actively running.
// The Recent generation progress being made will be saved before returning.
func (g *generator) stop() {
if !g.running {
log.Debug("Snapshot generation is not running")
return
}
ch := make(chan struct{})
g.abort <- ch
<-ch
g.running = false
}
// completed returns the flag indicating if the whole generation is done.
func (g *generator) completed() bool {
progress := g.progressMarker()
return progress == nil
}
// progressMarker returns the current generation progress marker. It may slightly
// lag behind the actual generation position, as the progress field is only updated
// when checkAndFlush is called. The only effect is that some generated states
// may be refused for serving.
func (g *generator) progressMarker() []byte {
g.lock.RLock()
defer g.lock.RUnlock()
return g.progress
}
// splitMarker is an internal helper which splits the generation progress marker
// into two parts.
func splitMarker(marker []byte) ([]byte, []byte) {
var accMarker []byte
if len(marker) > 0 {
accMarker = marker[:common.HashLength]
}
return accMarker, marker
}
// generateSnapshot regenerates a brand-new snapshot based on an existing state
// database and head block asynchronously. The snapshot is returned immediately
// and generation is continued in the background until done.
func generateSnapshot(triedb *Database, root common.Hash, noBuild bool) *diskLayer {
// Create a new disk layer with an initialized state marker at zero
var (
stats = &generatorStats{start: time.Now()}
genMarker = []byte{} // Initialized but empty!
)
dl := newDiskLayer(root, 0, triedb, nil, nil, newBuffer(triedb.config.WriteBufferSize, nil, nil, 0))
dl.setGenerator(newGenerator(triedb.diskdb, noBuild, genMarker, stats))
if !noBuild {
dl.generator.run(root)
log.Info("Started snapshot generation", "root", root)
}
return dl
}
// journalProgress persists the generator stats into the database to resume later.
func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorStats) {
// Write out the generator marker. Note it's a standalone disk layer generator
// which is not mixed with journal. It's ok if the generator is persisted while
// journal is not.
entry := journalGenerator{
Done: marker == nil,
Marker: marker,
}
if stats != nil {
entry.Accounts = stats.accounts
entry.Slots = stats.slots
entry.Storage = uint64(stats.storage)
}
blob, err := rlp.EncodeToBytes(entry)
if err != nil {
panic(err) // Cannot happen, here to catch dev errors
}
var logstr string
switch {
case marker == nil:
logstr = "done"
case bytes.Equal(marker, []byte{}):
logstr = "empty"
case len(marker) == common.HashLength:
logstr = fmt.Sprintf("%#x", marker)
default:
logstr = fmt.Sprintf("%#x:%#x", marker[:common.HashLength], marker[common.HashLength:])
}
log.Debug("Journalled generator progress", "progress", logstr)
rawdb.WriteSnapshotGenerator(db, blob)
}
// proofResult contains the output of range proving which can be used
// for further processing regardless if it is successful or not.
type proofResult struct {
keys [][]byte // The key set of all elements being iterated, even proving is failed
vals [][]byte // The val set of all elements being iterated, even proving is failed
diskMore bool // Set when the database has extra snapshot states since last iteration
trieMore bool // Set when the trie has extra snapshot states(only meaningful for successful proving)
proofErr error // Indicator whether the given state range is valid or not
tr *trie.Trie // The trie, in case the trie was resolved by the prover (may be nil)
}
// valid returns the indicator that range proof is successful or not.
func (result *proofResult) valid() bool {
return result.proofErr == nil
}
// last returns the last verified element key regardless of whether the range proof is
// successful or not. Nil is returned if nothing involved in the proving.
func (result *proofResult) last() []byte {
var last []byte
if len(result.keys) > 0 {
last = result.keys[len(result.keys)-1]
}
return last
}
// forEach iterates all the visited elements and applies the given callback on them.
// The iteration is aborted if the callback returns non-nil error.
func (result *proofResult) forEach(callback func(key []byte, val []byte) error) error {
for i := 0; i < len(result.keys); i++ {
key, val := result.keys[i], result.vals[i]
if err := callback(key, val); err != nil {
return err
}
}
return nil
}
// proveRange proves the snapshot segment with particular prefix is "valid".
// The iteration start point will be assigned if the iterator is restored from
// the last interruption. Max will be assigned in order to limit the maximum
// amount of data involved in each iteration.
//
// The proof result will be returned if the range proving is finished, otherwise
// the error will be returned to abort the entire procedure.
func (g *generator) proveRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
var (
keys [][]byte
vals [][]byte
proof = rawdb.NewMemoryDatabase()
diskMore = false
iter = ctx.iterator(kind)
start = time.Now()
min = append(prefix, origin...)
)
for iter.Next() {
// Ensure the iterated item is always equal or larger than the given origin.
key := iter.Key()
if bytes.Compare(key, min) < 0 {
return nil, errors.New("invalid iteration position")
}
// Ensure the iterated item still fall in the specified prefix. If
// not which means the items in the specified area are all visited.
// Move the iterator a step back since we iterate one extra element
// out.
if !bytes.Equal(key[:len(prefix)], prefix) {
iter.Hold()
break
}
// Break if we've reached the max size, and signal that we're not
// done yet. Move the iterator a step back since we iterate one
// extra element out.
if len(keys) == max {
iter.Hold()
diskMore = true
break
}
keys = append(keys, common.CopyBytes(key[len(prefix):]))
if valueConvertFn == nil {
vals = append(vals, common.CopyBytes(iter.Value()))
} else {
val, err := valueConvertFn(iter.Value())
if err != nil {
// Special case, the state data is corrupted (invalid slim-format account),
// don't abort the entire procedure directly. Instead, let the fallback
// generation to heal the invalid data.
//
// Here append the original value to ensure that the number of key and
// value are aligned.
vals = append(vals, common.CopyBytes(iter.Value()))
log.Error("Failed to convert account state data", "err", err)
} else {
vals = append(vals, val)
}
}
}
// Update metrics for database iteration and merkle proving
if kind == snapStorage {
storageSnapReadCounter.Inc(time.Since(start).Nanoseconds())
} else {
accountSnapReadCounter.Inc(time.Since(start).Nanoseconds())
}
defer func(start time.Time) {
if kind == snapStorage {
storageProveCounter.Inc(time.Since(start).Nanoseconds())
} else {
accountProveCounter.Inc(time.Since(start).Nanoseconds())
}
}(time.Now())
// The snap state is exhausted, pass the entire key/val set for verification
root := trieId.Root
if origin == nil && !diskMore {
stackTr := trie.NewStackTrie(nil)
for i, key := range keys {
if err := stackTr.Update(key, vals[i]); err != nil {
return nil, err
}
}
if gotRoot := stackTr.Hash(); gotRoot != root {
return &proofResult{
keys: keys,
vals: vals,
proofErr: fmt.Errorf("wrong root: have %#x want %#x", gotRoot, root),
}, nil
}
return &proofResult{keys: keys, vals: vals}, nil
}
// Snap state is chunked, generate edge proofs for verification.
tr, err := trie.New(trieId, &diskStore{db: g.db})
if err != nil {
log.Info("Trie missing, snapshotting paused", "state", ctx.root, "kind", kind, "root", trieId.Root)
return nil, errMissingTrie
}
// Generate the Merkle proofs for the first and last element
if origin == nil {
origin = common.Hash{}.Bytes()
}
if err := tr.Prove(origin, proof); err != nil {
log.Debug("Failed to prove range", "kind", kind, "origin", origin, "err", err)
return &proofResult{
keys: keys,
vals: vals,
diskMore: diskMore,
proofErr: err,
tr: tr,
}, nil
}
if len(keys) > 0 {
if err := tr.Prove(keys[len(keys)-1], proof); err != nil {
log.Debug("Failed to prove range", "kind", kind, "last", keys[len(keys)-1], "err", err)
return &proofResult{
keys: keys,
vals: vals,
diskMore: diskMore,
proofErr: err,
tr: tr,
}, nil
}
}
// Verify the snapshot segment with range prover, ensure that all flat states
// in this range correspond to merkle trie.
cont, err := trie.VerifyRangeProof(root, origin, keys, vals, proof)
return &proofResult{
keys: keys,
vals: vals,
diskMore: diskMore,
trieMore: cont,
proofErr: err,
tr: tr},
nil
}
// onStateCallback is a function that is called by generateRange, when processing a range of
// accounts or storage slots. For each element, the callback is invoked.
//
// - If 'delete' is true, then this element (and potential slots) needs to be deleted from the snapshot.
// - If 'write' is true, then this element needs to be updated with the 'val'.
// - If 'write' is false, then this element is already correct, and needs no update.
// The 'val' is the canonical encoding of the value (not the slim format for accounts)
//
// However, for accounts, the storage trie of the account needs to be checked. Also,
// dangling storages(storage exists but the corresponding account is missing) need to
// be cleaned up.
type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
// generateRange generates the state segment with particular prefix. Generation can
// either verify the correctness of existing state through range-proof and skip
// generation, or iterate trie to regenerate state on demand.
func (g *generator) generateRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
// Use range prover to check the validity of the flat state in the range
result, err := g.proveRange(ctx, trieId, prefix, kind, origin, max, valueConvertFn)
if err != nil {
return false, nil, err
}
last := result.last()
// Construct contextual logger
logCtx := []interface{}{"kind", kind, "prefix", hexutil.Encode(prefix)}
if len(origin) > 0 {
logCtx = append(logCtx, "origin", hexutil.Encode(origin))
}
logger := log.New(logCtx...)
// The range prover says the range is correct, skip trie iteration
if result.valid() {
successfulRangeProofMeter.Mark(1)
logger.Trace("Proved state range", "last", hexutil.Encode(last))
// The verification is passed, process each state with the given
// callback function. If this state represents a contract, the
// corresponding storage check will be performed in the callback
if err := result.forEach(func(key []byte, val []byte) error { return onState(key, val, false, false) }); err != nil {
return false, nil, err
}
// Only abort the iteration when both database and trie are exhausted
return !result.diskMore && !result.trieMore, last, nil
}
logger.Trace("Detected outdated state range", "last", hexutil.Encode(last), "err", result.proofErr)
failedRangeProofMeter.Mark(1)
// Special case, the entire trie is missing. In the original trie scheme,
// all the duplicated subtries will be filtered out (only one copy of data
// will be stored). While in the snapshot model, all the storage tries
// belong to different contracts will be kept even they are duplicated.
// Track it to a certain extent remove the noise data used for statistics.
if origin == nil && last == nil {
meter := missallAccountMeter
if kind == snapStorage {
meter = missallStorageMeter
}
meter.Mark(1)
}
// We use the snap data to build up a cache which can be used by the
// main account trie as a primary lookup when resolving hashes
var resolver trie.NodeResolver
if len(result.keys) > 0 {
tr := trie.NewEmpty(nil)
for i, key := range result.keys {
tr.Update(key, result.vals[i])
}
_, nodes := tr.Commit(false)
hashSet := nodes.HashSet()
resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte {
return hashSet[hash]
}
}
// Construct the trie for state iteration, reuse the trie
// if it's already opened with some nodes resolved.
tr := result.tr
if tr == nil {
tr, err = trie.New(trieId, &diskStore{db: g.db})
if err != nil {
log.Info("Trie missing, snapshotting paused", "state", ctx.root, "kind", kind, "root", trieId.Root)
return false, nil, errMissingTrie
}
}
var (
trieMore bool
kvkeys, kvvals = result.keys, result.vals
// counters
count = 0 // number of states delivered by iterator
created = 0 // states created from the trie
updated = 0 // states updated from the trie
deleted = 0 // states not in trie, but were in snapshot
untouched = 0 // states already correct
// timers
start = time.Now()
internal time.Duration
)
nodeIt, err := tr.NodeIterator(origin)
if err != nil {
return false, nil, err
}
nodeIt.AddResolver(resolver)
iter := trie.NewIterator(nodeIt)
for iter.Next() {
if last != nil && bytes.Compare(iter.Key, last) > 0 {
trieMore = true
break
}
count++
write := true
created++
for len(kvkeys) > 0 {
if cmp := bytes.Compare(kvkeys[0], iter.Key); cmp < 0 {
// delete the key
istart := time.Now()
if err := onState(kvkeys[0], nil, false, true); err != nil {
return false, nil, err
}
kvkeys = kvkeys[1:]
kvvals = kvvals[1:]
deleted++
internal += time.Since(istart)
continue
} else if cmp == 0 {
// the snapshot key can be overwritten
created--
if write = !bytes.Equal(kvvals[0], iter.Value); write {
updated++
} else {
untouched++
}
kvkeys = kvkeys[1:]
kvvals = kvvals[1:]
}
break
}
istart := time.Now()
if err := onState(iter.Key, iter.Value, write, false); err != nil {
return false, nil, err
}
internal += time.Since(istart)
}
if iter.Err != nil {
// Trie errors should never happen. Still, in case of a bug, expose the
// error here, as the outer code will presume errors are interrupts, not
// some deeper issues.
log.Error("State snapshotter failed to iterate trie", "err", iter.Err)
return false, nil, iter.Err
}
// Delete all stale snapshot states remaining
istart := time.Now()
for _, key := range kvkeys {
if err := onState(key, nil, false, true); err != nil {
return false, nil, err
}
deleted += 1
}
internal += time.Since(istart)
// Update metrics for counting trie iteration
if kind == snapStorage {
storageTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
} else {
accountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
}
logger.Trace("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last),
"count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted)
// If there are either more trie items, or there are more snap items
// (in the next segment), then we need to keep working
return !trieMore && !result.diskMore, last, nil
}
// checkAndFlush checks if an interruption signal is received or the
// batch size has exceeded the allowance.
func (g *generator) checkAndFlush(ctx *generatorContext, current []byte) error {
var abort chan struct{}
select {
case abort = <-g.abort:
default:
}
if ctx.batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
if bytes.Compare(current, g.progress) < 0 {
log.Error("Snapshot generator went backwards", "current", fmt.Sprintf("%x", current), "genMarker", fmt.Sprintf("%x", g.progress))
}
// Persist the progress marker regardless of whether the batch is empty or not.
// It may happen that all the flat states in the database are correct, so the
// generator indeed makes progress even if there is nothing to commit.
journalProgress(ctx.batch, current, g.stats)
// Flush out the database writes atomically
if err := ctx.batch.Write(); err != nil {
return err
}
ctx.batch.Reset()
// Update the generation progress marker
g.lock.Lock()
g.progress = current
g.lock.Unlock()
// Abort the generation if it's required
if abort != nil {
g.stats.log("Aborting snapshot generation", ctx.root, g.progress)
return newAbortErr(abort) // bubble up an error for interruption
}
// Don't hold the iterators too long, release them to let compactor works
ctx.reopenIterator(snapAccount)
ctx.reopenIterator(snapStorage)
}
if time.Since(ctx.logged) > 8*time.Second {
g.stats.log("Generating snapshot", ctx.root, g.progress)
ctx.logged = time.Now()
}
return nil
}
// generateStorages generates the missing storage slots of the specific contract.
// It's supposed to restart the generation from the given origin position.
func (g *generator) generateStorages(ctx *generatorContext, account common.Hash, storageRoot common.Hash, storeMarker []byte) error {
onStorage := func(key []byte, val []byte, write bool, delete bool) error {
defer func(start time.Time) {
storageWriteCounter.Inc(time.Since(start).Nanoseconds())
}(time.Now())
if delete {
rawdb.DeleteStorageSnapshot(ctx.batch, account, common.BytesToHash(key))
wipedStorageMeter.Mark(1)
return nil
}
if write {
rawdb.WriteStorageSnapshot(ctx.batch, account, common.BytesToHash(key), val)
generatedStorageMeter.Mark(1)
} else {
recoveredStorageMeter.Mark(1)
}
g.stats.storage += common.StorageSize(1 + 2*common.HashLength + len(val))
g.stats.slots++
// If we've exceeded our batch allowance or termination was requested, flush to disk
if err := g.checkAndFlush(ctx, append(account[:], key...)); err != nil {
return err
}
return nil
}
// Loop for re-generating the missing storage slots.
var origin = common.CopyBytes(storeMarker)
for {
id := trie.StorageTrieID(ctx.root, account, storageRoot)
exhausted, last, err := g.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
// Abort the procedure if the entire contract storage is generated
if exhausted {
break
}
if origin = increaseKey(last); origin == nil {
break // special case, the last is 0xffffffff...fff
}
}
return nil
}
// generateAccounts generates the missing snapshot accounts as well as their
// storage slots in the main trie. It's supposed to restart the generation
// from the given origin position.
func (g *generator) generateAccounts(ctx *generatorContext, accMarker []byte) error {
onAccount := func(key []byte, val []byte, write bool, delete bool) error {
// Make sure to clear all dangling storages before this account
account := common.BytesToHash(key)
g.stats.dangling += ctx.removeStorageBefore(account)
start := time.Now()
if delete {
rawdb.DeleteAccountSnapshot(ctx.batch, account)
wipedAccountMeter.Mark(1)
accountWriteCounter.Inc(time.Since(start).Nanoseconds())
ctx.removeStorageAt(account)
return nil
}
// Retrieve the current account and flatten it into the internal format
var acc types.StateAccount
if err := rlp.DecodeBytes(val, &acc); err != nil {
log.Crit("Invalid account encountered during snapshot creation", "err", err)
}
// If the account is not yet in-progress, write it out
if accMarker == nil || !bytes.Equal(account[:], accMarker) {
dataLen := len(val) // Approximate size, saves us a round of RLP-encoding
if !write {
if bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
dataLen -= 32
}
if acc.Root == types.EmptyRootHash {
dataLen -= 32
}
recoveredAccountMeter.Mark(1)
} else {
data := types.SlimAccountRLP(acc)
dataLen = len(data)
rawdb.WriteAccountSnapshot(ctx.batch, account, data)
generatedAccountMeter.Mark(1)
}
g.stats.storage += common.StorageSize(1 + common.HashLength + dataLen)
g.stats.accounts++
}
// If the snap generation goes here after interrupted, genMarker may go backward
// when last genMarker is consisted of accountHash and storageHash
marker := account[:]
if accMarker != nil && bytes.Equal(marker, accMarker) && len(g.progress) > common.HashLength {
marker = g.progress
}
// If we've exceeded our batch allowance or termination was requested, flush to disk
if err := g.checkAndFlush(ctx, marker); err != nil {
return err
}
accountWriteCounter.Inc(time.Since(start).Nanoseconds()) // let's count flush time as well
// If the iterated account is the contract, create a further loop to
// verify or regenerate the contract storage.
if acc.Root == types.EmptyRootHash {
ctx.removeStorageAt(account)
} else {
var storeMarker []byte
if accMarker != nil && bytes.Equal(account[:], accMarker) && len(g.progress) > common.HashLength {
storeMarker = g.progress[common.HashLength:]
}
if err := g.generateStorages(ctx, account, acc.Root, storeMarker); err != nil {
return err
}
}
// Some account processed, unmark the marker
accMarker = nil
return nil
}
origin := common.CopyBytes(accMarker)
for {
id := trie.StateTrieID(ctx.root)
exhausted, last, err := g.generateRange(ctx, id, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountCheckRange, onAccount, types.FullAccountRLP)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
origin = increaseKey(last)
// Last step, cleanup the storages after the last account.
// All the left storages should be treated as dangling.
if origin == nil || exhausted {
g.stats.dangling += ctx.removeRemainingStorage()
break
}
}
return nil
}
// generate is a background thread that iterates over the state and storage tries,
// constructing the state snapshot. All the arguments are purely for statistics
// gathering and logging, since the method surfs the blocks as they arrive, often
// being restarted.
func (g *generator) generate(ctx *generatorContext) {
g.stats.log("Resuming snapshot generation", ctx.root, g.progress)
defer ctx.close()
// Persist the initial marker and state snapshot root if progress is none
if len(g.progress) == 0 {
batch := g.db.NewBatch()
rawdb.WriteSnapshotRoot(batch, ctx.root)
journalProgress(batch, g.progress, g.stats)
if err := batch.Write(); err != nil {
log.Crit("Failed to write initialized state marker", "err", err)
}
}
// Initialize the global generator context. The snapshot iterators are
// opened at the interrupted position because the assumption is held
// that all the snapshot data are generated correctly before the marker.
// Even if the snapshot data is updated during the interruption (before
// or at the marker), the assumption is still held.
// For the account or storage slot at the interruption, they will be
// processed twice by the generator(they are already processed in the
// last run) but it's fine.
var (
accMarker, _ = splitMarker(g.progress)
abort chan struct{}
)
if err := g.generateAccounts(ctx, accMarker); err != nil {
// Extract the received interruption signal if exists
var aerr *abortErr
if errors.As(err, &aerr) {
abort = aerr.abort
}
// Aborted by internal error, wait the signal
if abort == nil {
abort = <-g.abort
}
close(abort)
return
}
// Snapshot fully generated, set the marker to nil.
// Note even there is nothing to commit, persist the
// generator anyway to mark the snapshot is complete.
journalProgress(ctx.batch, nil, g.stats)
if err := ctx.batch.Write(); err != nil {
log.Error("Failed to flush batch", "err", err)
abort = <-g.abort
close(abort)
return
}
ctx.batch.Reset()
log.Info("Generated snapshot", "accounts", g.stats.accounts, "slots", g.stats.slots,
"storage", g.stats.storage, "dangling", g.stats.dangling, "elapsed", common.PrettyDuration(time.Since(g.stats.start)))
// Update the generation progress marker
g.lock.Lock()
g.progress = nil
g.lock.Unlock()
close(g.done)
// Someone will be looking for us, wait it out
abort = <-g.abort
close(abort)
}
// increaseKey increase the input key by one bit. Return nil if the entire
// addition operation overflows.
func increaseKey(key []byte) []byte {
for i := len(key) - 1; i >= 0; i-- {
key[i]++
if key[i] != 0x0 {
return key
}
}
return nil
}
// abortErr wraps an interruption signal received to represent the
// generation is aborted by external processes.
type abortErr struct {
abort chan struct{}
}
func newAbortErr(abort chan struct{}) error {
return &abortErr{abort: abort}
}
func (err *abortErr) Error() string {
return "aborted"
}

766
triedb/pathdb/generate_test.go Normal file
View file

@ -0,0 +1,766 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pathdb
import (
"fmt"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/testrand"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/holiman/uint256"
)
func hashData(input []byte) common.Hash {
return crypto.Keccak256Hash(input)
}
type genTester struct {
diskdb ethdb.Database
db *Database
acctTrie *trie.Trie
nodes *trienode.MergedNodeSet
states *StateSetWithOrigin
}
func newGenTester() *genTester {
disk := rawdb.NewMemoryDatabase()
config := *Defaults
config.SnapshotNoBuild = true // no background generation
db := New(disk, &config, false)
tr, _ := trie.New(trie.StateTrieID(types.EmptyRootHash), db)
return &genTester{
diskdb: disk,
db: db,
acctTrie: tr,
nodes: trienode.NewMergedNodeSet(),
states: NewStateSetWithOrigin(nil, nil, nil, nil, false),
}
}
func (t *genTester) addTrieAccount(acckey string, acc *types.StateAccount) {
var (
addr = common.BytesToAddress([]byte(acckey))
key = hashData([]byte(acckey))
val, _ = rlp.EncodeToBytes(acc)
)
t.acctTrie.MustUpdate(key.Bytes(), val)
t.states.accountData[key] = val
t.states.accountOrigin[addr] = nil
}
func (t *genTester) addSnapAccount(acckey string, acc *types.StateAccount) {
key := hashData([]byte(acckey))
rawdb.WriteAccountSnapshot(t.diskdb, key, types.SlimAccountRLP(*acc))
}
func (t *genTester) addAccount(acckey string, acc *types.StateAccount) {
t.addTrieAccount(acckey, acc)
t.addSnapAccount(acckey, acc)
}
func (t *genTester) addSnapStorage(accKey string, keys []string, vals []string) {
accHash := hashData([]byte(accKey))
for i, key := range keys {
rawdb.WriteStorageSnapshot(t.diskdb, accHash, hashData([]byte(key)), []byte(vals[i]))
}
}
func (t *genTester) makeStorageTrie(accKey string, keys []string, vals []string, commit bool) common.Hash {
var (
owner = hashData([]byte(accKey))
addr = common.BytesToAddress([]byte(accKey))
id = trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash)
tr, _ = trie.New(id, t.db)
storages = make(map[common.Hash][]byte)
storageOrigins = make(map[common.Hash][]byte)
)
for i, k := range keys {
key := hashData([]byte(k))
tr.MustUpdate(key.Bytes(), []byte(vals[i]))
storages[key] = []byte(vals[i])
storageOrigins[key] = nil
}
if !commit {
return tr.Hash()
}
root, nodes := tr.Commit(false)
if nodes != nil {
t.nodes.Merge(nodes)
}
t.states.storageData[owner] = storages
t.states.storageOrigin[addr] = storageOrigins
return root
}
func (t *genTester) Commit() common.Hash {
root, nodes := t.acctTrie.Commit(true)
if nodes != nil {
t.nodes.Merge(nodes)
}
t.db.Update(root, types.EmptyRootHash, 0, t.nodes, t.states)
t.db.Commit(root, false)
return root
}
func (t *genTester) CommitAndGenerate() (common.Hash, *diskLayer) {
root := t.Commit()
dl := generateSnapshot(t.db, root, false)
return root, dl
}
// Tests that snapshot generation from an empty database.
func TestGeneration(t *testing.T) {
helper := newGenTester()
stRoot := helper.makeStorageTrie("", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
root, dl := helper.CommitAndGenerate()
if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
t.Fatalf("have %#x want %#x", have, want)
}
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
// Tests that snapshot generation with existent flat state, where the flat state
// contains some errors:
// - the contract with empty storage root but has storage entries in the disk
// - the contract with non empty storage root but empty storage slots
// - the contract(non-empty storage) misses some storage slots
// - miss in the beginning
// - miss in the middle
// - miss in the end
//
// - the contract(non-empty storage) has wrong storage slots
// - wrong slots in the beginning
// - wrong slots in the middle
// - wrong slots in the end
//
// - the contract(non-empty storage) has extra storage slots
// - extra slots in the beginning
// - extra slots in the middle
// - extra slots in the end
func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
helper := newGenTester()
// Account one, empty storage trie root but non-empty flat states
helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Account two, non-empty storage trie root but empty flat states
stRoot := helper.makeStorageTrie("acc-2", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
// Miss slots
{
// Account three, non-empty root but misses slots in the beginning
helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
// Account four, non-empty root but misses slots in the middle
helper.makeStorageTrie("acc-4", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
// Account five, non-empty root but misses slots in the end
helper.makeStorageTrie("acc-5", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
}
// Wrong storage slots
{
// Account six, non-empty root but wrong slots in the beginning
helper.makeStorageTrie("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
// Account seven, non-empty root but wrong slots in the middle
helper.makeStorageTrie("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
// Account eight, non-empty root but wrong slots in the end
helper.makeStorageTrie("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-8", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
// Account 9, non-empty root but rotated slots
helper.makeStorageTrie("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-9", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
}
// Extra storage slots
{
// Account 10, non-empty root but extra slots in the beginning
helper.makeStorageTrie("acc-10", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-10", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
// Account 11, non-empty root but extra slots in the middle
helper.makeStorageTrie("acc-11", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-11", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
// Account 12, non-empty root but extra slots in the end
helper.makeStorageTrie("acc-12", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-12", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
}
root, dl := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
// Tests that snapshot generation with existent flat state, where the flat state
// contains some errors:
// - miss accounts
// - wrong accounts
// - extra accounts
func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
helper := newGenTester()
// Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6]
helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie("acc-2", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.makeStorageTrie("acc-4", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
stRoot := helper.makeStorageTrie("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
// Missing accounts, only in the trie
{
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Beginning
helper.addTrieAccount("acc-4", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // Middle
helper.addTrieAccount("acc-6", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // End
}
// Wrong accounts
{
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")})
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
}
// Extra accounts, only in the snap
{
helper.addSnapAccount("acc-0", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // before the beginning
helper.addSnapAccount("acc-5", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: common.Hex2Bytes("0x1234")}) // Middle
helper.addSnapAccount("acc-7", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // after the end
}
root, dl := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateCorruptAccountTrie(t *testing.T) {
helper := newGenTester()
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie node and ensure the generator chokes
path := []byte{0xc}
if !rawdb.HasAccountTrieNode(helper.diskdb, path) {
t.Logf("Invalid node path to delete, %v", path)
}
rawdb.DeleteAccountTrieNode(helper.diskdb, path)
helper.db.tree.bottom().resetCache()
dl := generateSnapshot(helper.db, root, false)
select {
case <-dl.generator.done:
// Snapshot generation succeeded
t.Errorf("Snapshot generated against corrupt account trie")
case <-time.After(time.Second):
// Not generated fast enough, hopefully blocked inside on missing trie node fail
}
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateMissingStorageTrie(t *testing.T) {
var (
acc1 = hashData([]byte("acc-1"))
acc3 = hashData([]byte("acc-3"))
helper = newGenTester()
)
stRoot := helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root := helper.Commit()
// Delete storage trie root of account one and three.
rawdb.DeleteStorageTrieNode(helper.diskdb, acc1, nil)
rawdb.DeleteStorageTrieNode(helper.diskdb, acc3, nil)
helper.db.tree.bottom().resetCache()
dl := generateSnapshot(helper.db, root, false)
select {
case <-dl.generator.done:
// Snapshot generation succeeded
t.Errorf("Snapshot generated against corrupt storage trie")
case <-time.After(time.Second):
// Not generated fast enough, hopefully blocked inside on missing trie node fail
}
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateCorruptStorageTrie(t *testing.T) {
helper := newGenTester()
stRoot := helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root := helper.Commit()
// Delete a node in the storage trie.
path := []byte{0x4}
if !rawdb.HasStorageTrieNode(helper.diskdb, hashData([]byte("acc-1")), path) {
t.Logf("Invalid node path to delete, %v", path)
}
rawdb.DeleteStorageTrieNode(helper.diskdb, hashData([]byte("acc-1")), []byte{0x4})
if !rawdb.HasStorageTrieNode(helper.diskdb, hashData([]byte("acc-3")), path) {
t.Logf("Invalid node path to delete, %v", path)
}
rawdb.DeleteStorageTrieNode(helper.diskdb, hashData([]byte("acc-3")), []byte{0x4})
helper.db.tree.bottom().resetCache()
dl := generateSnapshot(helper.db, root, false)
select {
case <-dl.generator.done:
// Snapshot generation succeeded
t.Errorf("Snapshot generated against corrupt storage trie")
case <-time.After(time.Second):
// Not generated fast enough, hopefully blocked inside on missing trie node fail
}
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateWithExtraAccounts(t *testing.T) {
helper := newGenTester()
// Account one in the trie
stRoot := helper.makeStorageTrie("acc-1",
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true,
)
acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
helper.acctTrie.MustUpdate(hashData([]byte("acc-1")).Bytes(), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap
key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-4")), []byte("val-4"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
// Account two exists only in the snapshot
stRoot = helper.makeStorageTrie("acc-2",
[]string{"key-1", "key-2", "key-3", "key-4", "key-5"},
[]string{"val-1", "val-2", "val-3", "val-4", "val-5"},
true,
)
acc = &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ = rlp.EncodeToBytes(acc)
key = hashData([]byte("acc-2"))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
root := helper.Commit()
// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
t.Fatalf("expected snap storage to exist")
}
dl := generateSnapshot(helper.db, root, false)
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
// If we now inspect the snap db, there should exist no extraneous storage items
if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data))
}
}
func TestGenerateWithManyExtraAccounts(t *testing.T) {
helper := newGenTester()
// Account one in the trie
stRoot := helper.makeStorageTrie("acc-1",
[]string{"key-1", "key-2", "key-3"},
[]string{"val-1", "val-2", "val-3"},
true,
)
acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
helper.acctTrie.MustUpdate(hashData([]byte("acc-1")).Bytes(), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
// Identical in the snap
key := hashData([]byte("acc-1"))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
// 100 accounts exist only in snapshot
for i := 0; i < 1000; i++ {
acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
}
_, dl := helper.CommitAndGenerate()
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
helper := newGenTester()
acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
acctHashA := hashData([]byte("acc-1"))
acctHashB := hashData([]byte("acc-2"))
helper.acctTrie.MustUpdate(acctHashA.Bytes(), val)
helper.acctTrie.MustUpdate(acctHashB.Bytes(), val)
rawdb.WriteAccountSnapshot(helper.diskdb, acctHashA, val)
rawdb.WriteAccountSnapshot(helper.diskdb, acctHashB, val)
for i := 0; i < 16; i++ {
rawdb.WriteAccountSnapshot(helper.diskdb, common.Hash{byte(i)}, val)
}
_, dl := helper.CommitAndGenerate()
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateWithMalformedStateData(t *testing.T) {
helper := newGenTester()
acctHash := hashData([]byte("acc"))
acc := &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
helper.acctTrie.MustUpdate(acctHash.Bytes(), val)
junk := make([]byte, 100)
copy(junk, []byte{0xde, 0xad})
rawdb.WriteAccountSnapshot(helper.diskdb, acctHash, junk)
for i := 0; i < 16; i++ {
rawdb.WriteAccountSnapshot(helper.diskdb, common.Hash{byte(i)}, junk)
}
_, dl := helper.CommitAndGenerate()
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateFromEmptySnap(t *testing.T) {
helper := newGenTester()
for i := 0; i < 400; i++ {
stRoot := helper.makeStorageTrie(fmt.Sprintf("acc-%d", i), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount(fmt.Sprintf("acc-%d", i), &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
}
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
select {
case <-snap.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
snap.generator.stop()
}
func TestGenerateWithIncompleteStorage(t *testing.T) {
helper := newGenTester()
stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
// We add 8 accounts, each one is missing exactly one of the storage slots. This means
// we don't have to order the keys and figure out exactly which hash-key winds up
// on the sensitive spots at the boundaries
for i := 0; i < 8; i++ {
accKey := fmt.Sprintf("acc-%d", i)
stRoot := helper.makeStorageTrie(accKey, stKeys, stVals, true)
helper.addAccount(accKey, &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
var moddedKeys []string
var moddedVals []string
for ii := 0; ii < 8; ii++ {
if ii != i {
moddedKeys = append(moddedKeys, stKeys[ii])
moddedVals = append(moddedVals, stVals[ii])
}
}
helper.addSnapStorage(accKey, moddedKeys, moddedVals)
}
root, dl := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func incKey(key []byte) []byte {
for i := len(key) - 1; i >= 0; i-- {
key[i]++
if key[i] != 0x0 {
break
}
}
return key
}
func decKey(key []byte) []byte {
for i := len(key) - 1; i >= 0; i-- {
key[i]--
if key[i] != 0xff {
break
}
}
return key
}
func populateDangling(disk ethdb.KeyValueStore) {
populate := func(accountHash common.Hash, keys []string, vals []string) {
for i, key := range keys {
rawdb.WriteStorageSnapshot(disk, accountHash, hashData([]byte(key)), []byte(vals[i]))
}
}
// Dangling storages of the "first" account
populate(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Dangling storages of the "last" account
populate(common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Dangling storages around the account 1
hash := decKey(hashData([]byte("acc-1")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
hash = incKey(hashData([]byte("acc-1")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Dangling storages around the account 2
hash = decKey(hashData([]byte("acc-2")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
hash = incKey(hashData([]byte("acc-2")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Dangling storages around the account 3
hash = decKey(hashData([]byte("acc-3")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
hash = incKey(hashData([]byte("acc-3")).Bytes())
populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
// Dangling storages of the random account
populate(testrand.Hash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
populate(testrand.Hash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
populate(testrand.Hash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
}
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newGenTester()
stRoot := helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
populateDangling(helper.diskdb)
_, dl := helper.CommitAndGenerate()
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newGenTester()
stRoot := helper.makeStorageTrie("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
helper.addTrieAccount("acc-2", &types.StateAccount{Balance: uint256.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
helper.makeStorageTrie("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &types.StateAccount{Balance: uint256.NewInt(3), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
populateDangling(helper.diskdb)
_, dl := helper.CommitAndGenerate()
select {
case <-dl.generator.done:
// Snapshot generation succeeded
case <-time.After(3 * time.Second):
t.Errorf("Snapshot generation failed")
}
// TODO(rjl493456442) enable the snapshot tests
// checkSnapRoot(t, snap, root)
// Signal abortion to the generator and wait for it to tear down
dl.generator.stop()
}

18
triedb/pathdb/iterator.go
View file

@ -91,15 +91,14 @@ type diffAccountIterator struct {
}
// newDiffAccountIterator creates an account iterator over the given state set.
func newDiffAccountIterator(seek common.Hash, states *stateSet, fn loadAccount) AccountIterator {
func newDiffAccountIterator(seek common.Hash, accountList []common.Hash, fn loadAccount) AccountIterator {
// Seek out the requested starting account
hashes := states.accountList()
index := sort.Search(len(hashes), func(i int) bool {
return bytes.Compare(seek[:], hashes[i][:]) <= 0
index := sort.Search(len(accountList), func(i int) bool {
return bytes.Compare(seek[:], accountList[i][:]) <= 0
})
// Assemble and returned the already seeked iterator
return &diffAccountIterator{
keys: hashes[index:],
keys: accountList[index:],
loadFn: fn,
}
}
@ -236,15 +235,14 @@ type diffStorageIterator struct {
}
// newDiffStorageIterator creates a storage iterator over a single diff layer.
func newDiffStorageIterator(account common.Hash, seek common.Hash, states *stateSet, fn loadStorage) StorageIterator {
hashes := states.storageList(account)
index := sort.Search(len(hashes), func(i int) bool {
return bytes.Compare(seek[:], hashes[i][:]) <= 0
func newDiffStorageIterator(account common.Hash, seek common.Hash, storageList []common.Hash, fn loadStorage) StorageIterator {
index := sort.Search(len(storageList), func(i int) bool {
return bytes.Compare(seek[:], storageList[i][:]) <= 0
})
// Assemble and returned the already seeked iterator
return &diffStorageIterator{
account: account,
keys: hashes[index:],
keys: storageList[index:],
loadFn: fn,
}
}

36
triedb/pathdb/iterator_binary.go
View file

@ -45,6 +45,12 @@ type binaryIterator struct {
// accounts in a slow, but easily verifiable way. Note this function is used
// for initialization, use `newBinaryAccountIterator` as the API.
func (dl *diskLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator {
// The state set in the disk layer is mutable, hold the lock before obtaining
// the account list to prevent concurrent map iteration and write.
dl.lock.RLock()
accountList := dl.buffer.states.accountList()
dl.lock.RUnlock()
// Create two iterators for state buffer and the persistent state in disk
// respectively and combine them as a binary iterator.
l := &binaryIterator{
@ -54,7 +60,7 @@ func (dl *diskLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator
// The account key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffAccountIterator(seek, dl.buffer.states, nil),
a: newDiffAccountIterator(seek, accountList, nil),
b: newDiskAccountIterator(dl.db.diskdb, seek),
}
l.aDone = !l.a.Next()
@ -68,6 +74,9 @@ func (dl *diskLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator
func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator {
parent, ok := dl.parent.(*diffLayer)
if !ok {
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
accountList := dl.states.stateSet.accountList()
l := &binaryIterator{
// The account loader function is unnecessary; the account key list
// produced by the supplied state set alone is sufficient for iteration.
@ -75,13 +84,16 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator
// The account key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffAccountIterator(seek, dl.states.stateSet, nil),
a: newDiffAccountIterator(seek, accountList, nil),
b: dl.parent.(*diskLayer).initBinaryAccountIterator(seek),
}
l.aDone = !l.a.Next()
l.bDone = !l.b.Next()
return l
}
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
accountList := dl.states.stateSet.accountList()
l := &binaryIterator{
// The account loader function is unnecessary; the account key list
// produced by the supplied state set alone is sufficient for iteration.
@ -89,7 +101,7 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator
// The account key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffAccountIterator(seek, dl.states.stateSet, nil),
a: newDiffAccountIterator(seek, accountList, nil),
b: parent.initBinaryAccountIterator(seek),
}
l.aDone = !l.a.Next()
@ -101,6 +113,12 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) *binaryIterator
// storage slots in a slow, but easily verifiable way. Note this function is used
// for initialization, use `newBinaryStorageIterator` as the API.
func (dl *diskLayer) initBinaryStorageIterator(account common.Hash, seek common.Hash) *binaryIterator {
// The state set in the disk layer is mutable, hold the lock before obtaining
// the storage list to prevent concurrent map iteration and write.
dl.lock.RLock()
storageList := dl.buffer.states.storageList(account)
dl.lock.RUnlock()
// Create two iterators for state buffer and the persistent state in disk
// respectively and combine them as a binary iterator.
l := &binaryIterator{
@ -110,7 +128,7 @@ func (dl *diskLayer) initBinaryStorageIterator(account common.Hash, seek common.
// The storage key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffStorageIterator(account, seek, dl.buffer.states, nil),
a: newDiffStorageIterator(account, seek, storageList, nil),
b: newDiskStorageIterator(dl.db.diskdb, account, seek),
}
l.aDone = !l.a.Next()
@ -124,6 +142,9 @@ func (dl *diskLayer) initBinaryStorageIterator(account common.Hash, seek common.
func (dl *diffLayer) initBinaryStorageIterator(account common.Hash, seek common.Hash) *binaryIterator {
parent, ok := dl.parent.(*diffLayer)
if !ok {
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
storageList := dl.states.stateSet.storageList(account)
l := &binaryIterator{
// The storage loader function is unnecessary; the storage key list
// produced by the supplied state set alone is sufficient for iteration.
@ -131,13 +152,16 @@ func (dl *diffLayer) initBinaryStorageIterator(account common.Hash, seek common.
// The storage key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffStorageIterator(account, seek, dl.states.stateSet, nil),
a: newDiffStorageIterator(account, seek, storageList, nil),
b: dl.parent.(*diskLayer).initBinaryStorageIterator(account, seek),
}
l.aDone = !l.a.Next()
l.bDone = !l.b.Next()
return l
}
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
storageList := dl.states.stateSet.storageList(account)
l := &binaryIterator{
// The storage loader function is unnecessary; the storage key list
// produced by the supplied state set alone is sufficient for iteration.
@ -145,7 +169,7 @@ func (dl *diffLayer) initBinaryStorageIterator(account common.Hash, seek common.
// The storage key list for iteration is deterministic once the iterator
// is constructed, no matter the referenced disk layer is stale or not
// later.
a: newDiffStorageIterator(account, seek, dl.states.stateSet, nil),
a: newDiffStorageIterator(account, seek, storageList, nil),
b: parent.initBinaryStorageIterator(account, seek),
}
l.aDone = !l.a.Next()

25
triedb/pathdb/iterator_fast.go
View file

@ -76,11 +76,17 @@ func newFastIterator(db *Database, root common.Hash, account common.Hash, seek c
if accountIterator {
switch dl := current.(type) {
case *diskLayer:
// The state set in the disk layer is mutable, hold the lock before obtaining
// the account list to prevent concurrent map iteration and write.
dl.lock.RLock()
accountList := dl.buffer.states.accountList()
dl.lock.RUnlock()
fi.iterators = append(fi.iterators, &weightedIterator{
// The state set in the disk layer is mutable, and the entire state becomes stale
// if a diff layer above is merged into it. Therefore, staleness must be checked,
// and the storage slot should be retrieved with read lock protection.
it: newDiffAccountIterator(seek, dl.buffer.states, func(hash common.Hash) ([]byte, error) {
it: newDiffAccountIterator(seek, accountList, func(hash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
@ -98,19 +104,26 @@ func newFastIterator(db *Database, root common.Hash, account common.Hash, seek c
case *diffLayer:
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
accountList := dl.states.accountList()
fi.iterators = append(fi.iterators, &weightedIterator{
it: newDiffAccountIterator(seek, dl.states.stateSet, dl.states.mustAccount),
it: newDiffAccountIterator(seek, accountList, dl.states.mustAccount),
priority: depth,
})
}
} else {
switch dl := current.(type) {
case *diskLayer:
// The state set in the disk layer is mutable, hold the lock before obtaining
// the storage list to prevent concurrent map iteration and write.
dl.lock.RLock()
storageList := dl.buffer.states.storageList(account)
dl.lock.RUnlock()
fi.iterators = append(fi.iterators, &weightedIterator{
// The state set in the disk layer is mutable, and the entire state becomes stale
// if a diff layer above is merged into it. Therefore, staleness must be checked,
// and the storage slot should be retrieved with read lock protection.
it: newDiffStorageIterator(account, seek, dl.buffer.states, func(addrHash common.Hash, slotHash common.Hash) ([]byte, error) {
it: newDiffStorageIterator(account, seek, storageList, func(addrHash common.Hash, slotHash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
@ -126,10 +139,14 @@ func newFastIterator(db *Database, root common.Hash, account common.Hash, seek c
priority: depth + 1,
})
case *diffLayer:
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
storageList := dl.states.storageList(account)
// The state set in diff layer is immutable and will never be stale,
// so the read lock protection is unnecessary.
fi.iterators = append(fi.iterators, &weightedIterator{
it: newDiffStorageIterator(account, seek, dl.states.stateSet, dl.states.mustStorage),
it: newDiffStorageIterator(account, seek, storageList, dl.states.mustStorage),
priority: depth,
})
}

215
triedb/pathdb/iterator_test.go
View file

@ -132,18 +132,15 @@ func TestAccountIteratorBasics(t *testing.T) {
}
}
states := newStates(accounts, storage, false)
it := newDiffAccountIterator(common.Hash{}, states, nil)
it := newDiffAccountIterator(common.Hash{}, states.accountList(), nil)
verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator
// TODO reenable these tests once the persistent state iteration
// is implemented.
//db := rawdb.NewMemoryDatabase()
//batch := db.NewBatch()
//states.write(db, batch, nil, nil)
//batch.Write()
//it = newDiskAccountIterator(db, common.Hash{})
//verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator
db := rawdb.NewMemoryDatabase()
batch := db.NewBatch()
states.write(batch, nil, nil)
batch.Write()
it = newDiskAccountIterator(db, common.Hash{})
verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator
}
// TestStorageIteratorBasics tests some simple single-layer(diff and disk) iteration for storage
@ -173,21 +170,18 @@ func TestStorageIteratorBasics(t *testing.T) {
}
states := newStates(accounts, storage, false)
for account := range accounts {
it := newDiffStorageIterator(account, common.Hash{}, states, nil)
it := newDiffStorageIterator(account, common.Hash{}, states.storageList(account), nil)
verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator
}
// TODO reenable these tests once the persistent state iteration
// is implemented.
//db := rawdb.NewMemoryDatabase()
//batch := db.NewBatch()
//states.write(db, batch, nil, nil)
//batch.Write()
//for account := range accounts {
// it := newDiskStorageIterator(db, account, common.Hash{})
// verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator
//}
db := rawdb.NewMemoryDatabase()
batch := db.NewBatch()
states.write(batch, nil, nil)
batch.Write()
for account := range accounts {
it := newDiskStorageIterator(db, account, common.Hash{})
verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator
}
}
type testIterator struct {
@ -263,7 +257,7 @@ func TestAccountIteratorTraversal(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 0, trienode.NewMergedNodeSet(),
@ -279,7 +273,7 @@ func TestAccountIteratorTraversal(t *testing.T) {
head := db.tree.get(common.HexToHash("0x04"))
// singleLayer: 0xcc, 0xf0, 0xff
it := newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet, nil)
it := newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet.accountList(), nil)
verifyIterator(t, 3, it, verifyNothing)
// binaryIterator: 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xf0, 0xff
@ -290,19 +284,16 @@ func TestAccountIteratorTraversal(t *testing.T) {
verifyIterator(t, 7, it, verifyAccount)
it.Release()
// TODO reenable these tests once the persistent state iteration
// is implemented.
// Test after persist some bottom-most layers into the disk,
// the functionalities still work.
//db.tree.cap(common.HexToHash("0x04"), 2)
db.tree.cap(common.HexToHash("0x04"), 2)
//head = db.tree.get(common.HexToHash("0x04"))
//verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount)
//
//it, _ = db.AccountIterator(common.HexToHash("0x04"), common.Hash{})
//verifyIterator(t, 7, it, verifyAccount)
//it.Release()
head = db.tree.get(common.HexToHash("0x04"))
verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator(common.Hash{}), verifyAccount)
it, _ = db.AccountIterator(common.HexToHash("0x04"), common.Hash{})
verifyIterator(t, 7, it, verifyAccount)
it.Release()
}
func TestStorageIteratorTraversal(t *testing.T) {
@ -310,7 +301,7 @@ func TestStorageIteratorTraversal(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 0, trienode.NewMergedNodeSet(),
@ -326,7 +317,7 @@ func TestStorageIteratorTraversal(t *testing.T) {
head := db.tree.get(common.HexToHash("0x04"))
// singleLayer: 0x1, 0x2, 0x3
diffIter := newDiffStorageIterator(common.HexToHash("0xaa"), common.Hash{}, head.(*diffLayer).states.stateSet, nil)
diffIter := newDiffStorageIterator(common.HexToHash("0xaa"), common.Hash{}, head.(*diffLayer).states.stateSet.storageList(common.HexToHash("0xaa")), nil)
verifyIterator(t, 3, diffIter, verifyNothing)
// binaryIterator: 0x1, 0x2, 0x3, 0x4, 0x5, 0x6
@ -337,17 +328,14 @@ func TestStorageIteratorTraversal(t *testing.T) {
verifyIterator(t, 6, it, verifyStorage)
it.Release()
// TODO reenable these tests once the persistent state iteration
// is implemented.
// Test after persist some bottom-most layers into the disk,
// the functionalities still work.
//db.tree.cap(common.HexToHash("0x04"), 2)
//verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa")), verifyStorage)
//
//it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{})
//verifyIterator(t, 6, it, verifyStorage)
//it.Release()
db.tree.cap(common.HexToHash("0x04"), 2)
verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage)
it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 6, it, verifyStorage)
it.Release()
}
// TestAccountIteratorTraversalValues tests some multi-layer iteration, where we
@ -357,7 +345,7 @@ func TestAccountIteratorTraversalValues(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Create a batch of account sets to seed subsequent layers with
var (
@ -434,26 +422,38 @@ func TestAccountIteratorTraversalValues(t *testing.T) {
}
it.Release()
// TODO reenable these tests once the persistent state iteration
// is implemented.
// Test after persist some bottom-most layers into the disk,
// the functionalities still work.
//db.tree.cap(common.HexToHash("0x09"), 2)
//
//it, _ = db.AccountIterator(common.HexToHash("0x09"), common.Hash{})
//for it.Next() {
// hash := it.Hash()
// account, err := head.Account(hash)
// if err != nil {
// t.Fatalf("failed to retrieve expected account: %v", err)
// }
// want, _ := rlp.EncodeToBytes(account)
// if have := it.Account(); !bytes.Equal(want, have) {
// t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
// }
//}
//it.Release()
db.tree.cap(common.HexToHash("0x09"), 2)
// binaryIterator
head = db.tree.get(common.HexToHash("0x09"))
it = head.(*diffLayer).newBinaryAccountIterator(common.Hash{})
for it.Next() {
hash := it.Hash()
want, err := r.(*reader).AccountRLP(hash)
if err != nil {
t.Fatalf("failed to retrieve expected account: %v", err)
}
if have := it.Account(); !bytes.Equal(want, have) {
t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
}
}
it.Release()
// fastIterator
it, _ = db.AccountIterator(common.HexToHash("0x09"), common.Hash{})
for it.Next() {
hash := it.Hash()
want, err := r.(*reader).AccountRLP(hash)
if err != nil {
t.Fatalf("failed to retrieve expected account: %v", err)
}
if have := it.Account(); !bytes.Equal(want, have) {
t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
}
}
it.Release()
}
func TestStorageIteratorTraversalValues(t *testing.T) {
@ -461,7 +461,7 @@ func TestStorageIteratorTraversalValues(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
wrapStorage := func(storage map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte {
return map[common.Hash]map[common.Hash][]byte{
@ -543,25 +543,38 @@ func TestStorageIteratorTraversalValues(t *testing.T) {
}
it.Release()
// TODO reenable these tests once the persistent state iteration
// is implemented.
// Test after persist some bottom-most layers into the disk,
// the functionalities still work.
//db.tree.cap(common.HexToHash("0x09"), 2)
//
//it, _ = db.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{})
//for it.Next() {
// hash := it.Hash()
// want, err := head.Storage(common.HexToHash("0xaa"), hash)
// if err != nil {
// t.Fatalf("failed to retrieve expected slot: %v", err)
// }
// if have := it.Slot(); !bytes.Equal(want, have) {
// t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want)
// }
//}
//it.Release()
db.tree.cap(common.HexToHash("0x09"), 2)
// binaryIterator
head = db.tree.get(common.HexToHash("0x09"))
it = head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{})
for it.Next() {
hash := it.Hash()
want, err := r.Storage(common.HexToHash("0xaa"), hash)
if err != nil {
t.Fatalf("failed to retrieve expected account: %v", err)
}
if have := it.Slot(); !bytes.Equal(want, have) {
t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
}
}
it.Release()
// fastIterator
it, _ = db.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{})
for it.Next() {
hash := it.Hash()
want, err := r.Storage(common.HexToHash("0xaa"), hash)
if err != nil {
t.Fatalf("failed to retrieve expected storage slot: %v", err)
}
if have := it.Slot(); !bytes.Equal(want, have) {
t.Fatalf("hash %x: slot mismatch: have %x, want %x", hash, have, want)
}
}
it.Release()
}
// This testcase is notorious, all layers contain the exact same 200 accounts.
@ -581,7 +594,8 @@ func TestAccountIteratorLargeTraversal(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
for i := 1; i < 128; i++ {
parent := types.EmptyRootHash
if i == 1 {
@ -592,25 +606,22 @@ func TestAccountIteratorLargeTraversal(t *testing.T) {
}
// Iterate the entire stack and ensure everything is hit only once
head := db.tree.get(common.HexToHash("0x80"))
verifyIterator(t, 200, newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet, nil), verifyNothing)
verifyIterator(t, 200, newDiffAccountIterator(common.Hash{}, head.(*diffLayer).states.stateSet.accountList(), nil), verifyNothing)
verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(common.Hash{}), verifyAccount)
it, _ := db.AccountIterator(common.HexToHash("0x80"), common.Hash{})
verifyIterator(t, 200, it, verifyAccount)
it.Release()
// TODO reenable these tests once the persistent state iteration
// is implemented.
// Test after persist some bottom-most layers into the disk,
// the functionalities still work.
//db.tree.cap(common.HexToHash("0x80"), 2)
//
//verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(), verifyAccount)
//
//it, _ = db.AccountIterator(common.HexToHash("0x80"), common.Hash{})
//verifyIterator(t, 200, it, verifyAccount)
//it.Release()
db.tree.cap(common.HexToHash("0x80"), 2)
verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator(common.Hash{}), verifyAccount)
it, _ = db.AccountIterator(common.HexToHash("0x80"), common.Hash{})
verifyIterator(t, 200, it, verifyAccount)
it.Release()
}
// TestAccountIteratorFlattening tests what happens when we
@ -622,7 +633,7 @@ func TestAccountIteratorFlattening(t *testing.T) {
WriteBufferSize: 10 * 1024,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Create a stack of diffs on top
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -655,7 +666,7 @@ func TestAccountIteratorSeek(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil, nil, nil, false))
@ -727,7 +738,7 @@ func testStorageIteratorSeek(t *testing.T, newIterator func(db *Database, root,
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -799,7 +810,7 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -839,7 +850,7 @@ func TestStorageIteratorDeletions(t *testing.T) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -907,7 +918,7 @@ func testStaleIterator(t *testing.T, newIter func(db *Database, hash common.Hash
WriteBufferSize: 16 * 1024 * 1024,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
// [02 (disk), 03]
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -962,7 +973,7 @@ func BenchmarkAccountIteratorTraversal(b *testing.B) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
for i := 1; i <= 100; i++ {
parent := types.EmptyRootHash
@ -1057,7 +1068,7 @@ func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) {
WriteBufferSize: 0,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
// db.WaitGeneration()
db.waitGeneration()
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(makeAccounts(2000), nil, nil, nil, false))
for i := 2; i <= 100; i++ {

59
triedb/pathdb/journal.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
)
@ -90,6 +91,56 @@ func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
return head, nil
}
// journalGenerator is a disk layer entry containing the generator progress marker.
type journalGenerator struct {
// Indicator that whether the database was in progress of being wiped.
// It's deprecated but keep it here for backward compatibility.
Wiping bool
Done bool // Whether the generator finished creating the snapshot
Marker []byte
Accounts uint64
Slots uint64
Storage uint64
}
// loadGenerator loads the state generation progress marker from the database.
func loadGenerator(db ethdb.KeyValueReader, hash nodeHasher) (*journalGenerator, common.Hash, error) {
trieRoot, err := hash(rawdb.ReadAccountTrieNode(db, nil))
if err != nil {
return nil, common.Hash{}, err
}
// State generation progress marker is lost, rebuild it
blob := rawdb.ReadSnapshotGenerator(db)
if len(blob) == 0 {
log.Info("State snapshot generator is not found")
return nil, trieRoot, nil
}
// State generation progress marker is not compatible, rebuild it
var generator journalGenerator
if err := rlp.DecodeBytes(blob, &generator); err != nil {
log.Info("State snapshot generator is not compatible")
return nil, trieRoot, nil
}
// The state snapshot is inconsistent with the trie data and must
// be rebuilt.
//
// Note: The SnapshotRoot and SnapshotGenerator are always consistent
// with each other, both in the legacy state snapshot and the path database.
// Therefore, if the SnapshotRoot does not match the trie root,
// the entire generator is considered stale and must be discarded.
stateRoot := rawdb.ReadSnapshotRoot(db)
if trieRoot != stateRoot {
log.Info("State snapshot is not consistent", "trie", trieRoot, "state", stateRoot)
return nil, trieRoot, nil
}
// Slice null-ness is lost after rlp decoding, reset it back to empty
if !generator.Done && generator.Marker == nil {
generator.Marker = []byte{}
}
return &generator, trieRoot, nil
}
// loadLayers loads a pre-existing state layer backed by a key-value store.
func (db *Database) loadLayers() layer {
// Retrieve the root node of persistent state.
@ -109,7 +160,7 @@ func (db *Database) loadLayers() layer {
log.Info("Failed to load journal, discard it", "err", err)
}
// Return single layer with persistent state.
return newDiskLayer(root, rawdb.ReadPersistentStateID(db.diskdb), db, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0))
return newDiskLayer(root, rawdb.ReadPersistentStateID(db.diskdb), db, nil, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0))
}
// loadDiskLayer reads the binary blob from the layer journal, reconstructing
@ -141,7 +192,7 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
if err := states.decode(r); err != nil {
return nil, err
}
return newDiskLayer(root, id, db, nil, newBuffer(db.config.WriteBufferSize, &nodes, &states, id-stored)), nil
return newDiskLayer(root, id, db, nil, nil, newBuffer(db.config.WriteBufferSize, &nodes, &states, id-stored)), nil
}
// loadDiffLayer reads the next sections of a layer journal, reconstructing a new
@ -250,6 +301,10 @@ func (db *Database) Journal(root common.Hash) error {
} else { // disk layer only on noop runs (likely) or deep reorgs (unlikely)
log.Info("Persisting dirty state to disk", "root", root, "layers", disk.buffer.layers)
}
// Terminate the background state generation if it's active
if disk.generator != nil {
disk.generator.stop()
}
start := time.Now()
// Run the journaling

37
triedb/pathdb/metrics.go
View file

@ -24,6 +24,11 @@ var (
cleanNodeReadMeter = metrics.NewRegisteredMeter("pathdb/clean/node/read", nil)
cleanNodeWriteMeter = metrics.NewRegisteredMeter("pathdb/clean/node/write", nil)
cleanStateHitMeter = metrics.NewRegisteredMeter("pathdb/clean/state/hit", nil)
cleanStateMissMeter = metrics.NewRegisteredMeter("pathdb/clean/state/miss", nil)
cleanStateReadMeter = metrics.NewRegisteredMeter("pathdb/clean/state/read", nil)
cleanStateWriteMeter = metrics.NewRegisteredMeter("pathdb/clean/state/write", nil)
dirtyNodeHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/node/hit", nil)
dirtyNodeMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/node/miss", nil)
dirtyNodeReadMeter = metrics.NewRegisteredMeter("pathdb/dirty/node/read", nil)
@ -32,8 +37,13 @@ var (
stateAccountInexMeter = metrics.NewRegisteredMeter("pathdb/state/account/inex/total", nil)
stateStorageInexMeter = metrics.NewRegisteredMeter("pathdb/state/storage/inex/total", nil)
stateAccountInexDiskMeter = metrics.NewRegisteredMeter("pathdb/state/account/inex/disk", nil)
stateStorageInexDiskMeter = metrics.NewRegisteredMeter("pathdb/state/storage/inex/disk", nil)
stateAccountExistMeter = metrics.NewRegisteredMeter("pathdb/state/account/exist/total", nil)
stateStorageExistMeter = metrics.NewRegisteredMeter("pathdb/state/storage/exist/total", nil)
stateAccountExistDiskMeter = metrics.NewRegisteredMeter("pathdb/state/account/exist/disk", nil)
stateStorageExistDiskMeter = metrics.NewRegisteredMeter("pathdb/state/storage/exist/disk", nil)
dirtyStateHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/hit", nil)
dirtyStateMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/miss", nil)
@ -48,6 +58,8 @@ var (
commitTimeTimer = metrics.NewRegisteredResettingTimer("pathdb/commit/time", nil)
commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil)
commitAccountsMeter = metrics.NewRegisteredMeter("pathdb/commit/accounts", nil)
commitStoragesMeter = metrics.NewRegisteredMeter("pathdb/commit/slots", nil)
commitBytesMeter = metrics.NewRegisteredMeter("pathdb/commit/bytes", nil)
gcTrieNodeMeter = metrics.NewRegisteredMeter("pathdb/gc/node/count", nil)
@ -61,3 +73,28 @@ var (
historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
)
// Metrics in generation
var (
generatedAccountMeter = metrics.NewRegisteredMeter("pathdb/generation/account/generated", nil)
recoveredAccountMeter = metrics.NewRegisteredMeter("pathdb/generation/account/recovered", nil)
wipedAccountMeter = metrics.NewRegisteredMeter("pathdb/generation/account/wiped", nil)
missallAccountMeter = metrics.NewRegisteredMeter("pathdb/generation/account/missall", nil)
generatedStorageMeter = metrics.NewRegisteredMeter("pathdb/generation/storage/generated", nil)
recoveredStorageMeter = metrics.NewRegisteredMeter("pathdb/generation/storage/recovered", nil)
wipedStorageMeter = metrics.NewRegisteredMeter("pathdb/generation/storage/wiped", nil)
missallStorageMeter = metrics.NewRegisteredMeter("pathdb/generation/storage/missall", nil)
danglingStorageMeter = metrics.NewRegisteredMeter("pathdb/generation/storage/dangling", nil)
successfulRangeProofMeter = metrics.NewRegisteredMeter("pathdb/generation/proof/success", nil)
failedRangeProofMeter = metrics.NewRegisteredMeter("pathdb/generation/proof/failure", nil)
accountProveCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/account/prove", nil)
accountTrieReadCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/account/trieread", nil)
accountSnapReadCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/account/snapread", nil)
accountWriteCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/account/write", nil)
storageProveCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/storage/prove", nil)
storageTrieReadCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/storage/trieread", nil)
storageSnapReadCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/storage/snapread", nil)
storageWriteCounter = metrics.NewRegisteredCounter("pathdb/generation/duration/storage/write", nil)
storageCleanCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/clean", nil)
)

9
triedb/pathdb/states.go
View file

@ -23,8 +23,10 @@ import (
"slices"
"sync"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
@ -416,6 +418,11 @@ func (s *stateSet) decode(r *rlp.Stream) error {
return nil
}
// write flushes state mutations into the provided database batch as a whole.
func (s *stateSet) write(batch ethdb.Batch, genMarker []byte, clean *fastcache.Cache) (int, int) {
return writeStates(batch, genMarker, s.accountData, s.storageData, clean)
}
// reset clears all cached state data, including any optional sorted lists that
// may have been generated.
func (s *stateSet) reset() {
@ -427,8 +434,6 @@ func (s *stateSet) reset() {
}
// dbsize returns the approximate size for db write.
//
// nolint:unused
func (s *stateSet) dbsize() int {
m := len(s.accountData) * len(rawdb.SnapshotAccountPrefix)
for _, slots := range s.storageData {