forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-02-26 15:47:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into fix/get-logs-reverse-tags

Browse source
This commit is contained in:
Nikita Meshcheriakov 2026-01-20 14:23:17 -03:00 committed by GitHub
commit 8e48f336ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
114 changed files with 6098 additions and 1441 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
accounts/abi/bind/v2/dep_tree_test.go
View file

@ -158,10 +158,10 @@ func testLinkCase(tcInput linkTestCaseInput) error {
overrideAddrs = make(map[rune]common.Address)
)
// generate deterministic addresses for the override set.
rand.Seed(42)
rng := rand.New(rand.NewSource(42))
for contract := range tcInput.overrides {
var addr common.Address
rand.Read(addr[:])
rng.Read(addr[:])
overrideAddrs[contract] = addr
overridesAddrs[addr] = struct{}{}
}

3
accounts/keystore/presale.go
View file

@ -81,6 +81,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
*/
passBytes := []byte(password)
derivedKey := pbkdf2.Key(passBytes, passBytes, 2000, 16, sha256.New)
if len(cipherText)%aes.BlockSize != 0 {
return nil, errors.New("ciphertext must be a multiple of block size")
}
plainText, err := aesCBCDecrypt(derivedKey, cipherText, iv)
if err != nil {
return nil, err

4
accounts/scwallet/securechannel.go
View file

@ -300,6 +300,10 @@ func (s *SecureChannelSession) decryptAPDU(data []byte) ([]byte, error) {
return nil, err
}
if len(data) == 0 || len(data)%aes.BlockSize != 0 {
return nil, fmt.Errorf("invalid ciphertext length: %d", len(data))
}
ret := make([]byte, len(data))
crypter := cipher.NewCBCDecrypter(a, s.iv)

32
circle.yml
View file

@ -1,32 +0,0 @@
machine:
services:
- docker
dependencies:
cache_directories:
- "~/.ethash" # Cache the ethash DAG generated by hive for consecutive builds
- "~/.docker" # Cache all docker images manually to avoid lengthy rebuilds
override:
# Restore all previously cached docker images
- mkdir -p ~/.docker
- for img in `ls ~/.docker`; do docker load -i ~/.docker/$img; done
# Pull in and hive, restore cached ethash DAGs and do a dry run
- go get -u github.com/karalabe/hive
- (cd ~/.go_workspace/src/github.com/karalabe/hive && mkdir -p workspace/ethash/ ~/.ethash)
- (cd ~/.go_workspace/src/github.com/karalabe/hive && cp -r ~/.ethash/. workspace/ethash/)
- (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=NONE --test=. --sim=. --loglevel=6)
# Cache all the docker images and the ethash DAGs
- for img in `docker images | grep -v "^<none>" | tail -n +2 | awk '{print $1}'`; do docker save $img > ~/.docker/`echo $img | tr '/' ':'`.tar; done
- cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/ethash/. ~/.ethash
test:
override:
# Build Geth and move into a known folder
- make geth
- cp ./build/bin/geth $HOME/geth
# Run hive and move all generated logs into the public artifacts folder
- (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=go-ethereum:local --override=$HOME/geth --test=. --sim=.)
- cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/logs/* $CIRCLE_ARTIFACTS

3
cmd/geth/chaincmd.go
View file

@ -120,6 +120,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
utils.LogNoHistoryFlag,
utils.LogExportCheckpointsFlag,
utils.StateHistoryFlag,
utils.TrienodeHistoryFlag,
}, utils.DatabaseFlags, debug.Flags),
Before: func(ctx *cli.Context) error {
flags.MigrateGlobalFlags(ctx)
@ -296,7 +297,7 @@ func initGenesis(ctx *cli.Context) error {
triedb := utils.MakeTrieDatabase(ctx, stack, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
defer triedb.Close()
_, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
_, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides, nil)
if err != nil {
utils.Fatalf("Failed to write genesis block: %v", err)
}

2
cmd/geth/main.go
View file

@ -94,6 +94,7 @@ var (
utils.LogNoHistoryFlag,
utils.LogExportCheckpointsFlag,
utils.StateHistoryFlag,
utils.TrienodeHistoryFlag,
utils.LightKDFFlag,
utils.EthRequiredBlocksFlag,
utils.LegacyWhitelistFlag, // deprecated
@ -193,6 +194,7 @@ var (
utils.BatchResponseMaxSize,
utils.RPCTxSyncDefaultTimeoutFlag,
utils.RPCTxSyncMaxTimeoutFlag,
utils.RPCGlobalRangeLimitFlag,
}
metricsFlags = []cli.Flag{

3
cmd/keeper/go.mod
View file

@ -33,8 +33,9 @@ require (
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/sys v0.39.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

16
cmd/keeper/go.sum
View file

@ -63,8 +63,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ
github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA=
github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -96,12 +96,12 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4=
github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw=
github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
@ -118,8 +118,8 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=

38
cmd/utils/flags.go
View file

@ -295,6 +295,12 @@ var (
Value: ethconfig.Defaults.StateHistory,
Category: flags.StateCategory,
}
TrienodeHistoryFlag = &cli.Int64Flag{
Name: "history.trienode",
Usage: "Number of recent blocks to retain trienode history for, only relevant in state.scheme=path (default/negative = disabled, 0 = entire chain)",
Value: ethconfig.Defaults.TrienodeHistory,
Category: flags.StateCategory,
}
TransactionHistoryFlag = &cli.Uint64Flag{
Name: "history.transactions",
Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
@ -636,6 +642,12 @@ var (
Value: ethconfig.Defaults.TxSyncMaxTimeout,
Category: flags.APICategory,
}
RPCGlobalRangeLimitFlag = &cli.Uint64Flag{
Name: "rpc.rangelimit",
Usage: "Maximum block range (end - begin) allowed for range queries (0 = unlimited)",
Value: ethconfig.Defaults.RangeLimit,
Category: flags.APICategory,
}
// Authenticated RPC HTTP settings
AuthListenFlag = &cli.StringFlag{
Name: "authrpc.addr",
@ -1699,6 +1711,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(StateHistoryFlag.Name) {
cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
}
if ctx.IsSet(TrienodeHistoryFlag.Name) {
cfg.TrienodeHistory = ctx.Int64(TrienodeHistoryFlag.Name)
}
if ctx.IsSet(StateSchemeFlag.Name) {
cfg.StateScheme = ctx.String(StateSchemeFlag.Name)
}
@ -1753,6 +1768,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(RPCTxSyncMaxTimeoutFlag.Name) {
cfg.TxSyncMaxTimeout = ctx.Duration(RPCTxSyncMaxTimeoutFlag.Name)
}
if ctx.IsSet(RPCGlobalRangeLimitFlag.Name) {
cfg.RangeLimit = ctx.Uint64(RPCGlobalRangeLimitFlag.Name)
}
if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 {
// If snap-sync is requested, this flag is also required
if cfg.SyncMode == ethconfig.SnapSync {
@ -2097,6 +2115,7 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf
filterSystem := filters.NewFilterSystem(backend, filters.Config{
LogCacheSize: ethcfg.FilterLogCacheSize,
LogQueryLimit: ethcfg.LogQueryLimit,
RangeLimit: ethcfg.RangeLimit,
})
stack.RegisterAPIs([]rpc.API{{
Namespace: "eth",
@ -2299,15 +2318,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
Fatalf("%v", err)
}
options := &core.BlockChainConfig{
TrieCleanLimit: ethconfig.Defaults.TrieCleanCache,
NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name),
TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache,
ArchiveMode: ctx.String(GCModeFlag.Name) == "archive",
TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
SnapshotLimit: ethconfig.Defaults.SnapshotCache,
Preimages: ctx.Bool(CachePreimagesFlag.Name),
StateScheme: scheme,
StateHistory: ctx.Uint64(StateHistoryFlag.Name),
TrieCleanLimit: ethconfig.Defaults.TrieCleanCache,
NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name),
TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache,
ArchiveMode: ctx.String(GCModeFlag.Name) == "archive",
TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
SnapshotLimit: ethconfig.Defaults.SnapshotCache,
Preimages: ctx.Bool(CachePreimagesFlag.Name),
StateScheme: scheme,
StateHistory: ctx.Uint64(StateHistoryFlag.Name),
TrienodeHistory: ctx.Int64(TrienodeHistoryFlag.Name),
// Disable transaction indexing/unindexing.
TxLookupLimit: -1,

69
core/blockchain.go
View file

@ -177,6 +177,11 @@ type BlockChainConfig struct {
// If set to 0, all state histories across the entire chain will be retained;
StateHistory uint64
// Number of blocks from the chain head for which trienode histories are retained.
// If set to 0, all trienode histories across the entire chain will be retained;
// If set to -1, no trienode history will be retained;
TrienodeHistory int64
// State snapshot related options
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
SnapshotNoBuild bool // Whether the background generation is allowed
@ -255,6 +260,7 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config {
if cfg.StateScheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{
StateHistory: cfg.StateHistory,
TrienodeHistory: cfg.TrienodeHistory,
EnableStateIndexing: cfg.ArchiveMode,
TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024,
StateCleanSize: cfg.SnapshotLimit * 1024 * 1024,
@ -311,6 +317,7 @@ type BlockChain struct {
chainHeadFeed event.Feed
logsFeed event.Feed
blockProcFeed event.Feed
newPayloadFeed event.Feed // Feed for engine API newPayload events
blockProcCounter int32
scope event.SubscriptionScope
genesisBlock *types.Block
@ -366,7 +373,7 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine,
// yet. The corresponding chain config will be returned, either from the
// provided genesis or from the locally stored configuration if the genesis
// has already been initialized.
chainConfig, genesisHash, compatErr, err := SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides)
chainConfig, genesisHash, compatErr, err := SetupGenesisBlockWithOverride(db, triedb, genesis, cfg.Overrides, cfg.VmConfig.Tracer)
if err != nil {
return nil, err
}
@ -745,21 +752,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
if _, err := bc.setHeadBeyondRoot(head, 0, common.Hash{}, false); err != nil {
return err
}
// Send chain head event to update the transaction pool
header := bc.CurrentBlock()
if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
// In a pruned node the genesis block will not exist in the freezer.
// It should not happen that we set head to any other pruned block.
if header.Number.Uint64() > 0 {
// This should never happen. In practice, previously currentBlock
// contained the entire block whereas now only a "marker", so there
// is an ever so slight chance for a race we should handle.
log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
}
}
bc.chainHeadFeed.Send(ChainHeadEvent{Header: header})
return nil
return bc.sendChainHeadEvent()
}
// SetHeadWithTimestamp rewinds the local chain to a new head that has at max
@ -770,7 +763,12 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error {
if _, err := bc.setHeadBeyondRoot(0, timestamp, common.Hash{}, false); err != nil {
return err
}
// Send chain head event to update the transaction pool
return bc.sendChainHeadEvent()
}
// sendChainHeadEvent notifies all subscribers about the new chain head,
// checking first that the current block is actually available.
func (bc *BlockChain) sendChainHeadEvent() error {
header := bc.CurrentBlock()
if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
// In a pruned node the genesis block will not exist in the freezer.
@ -1650,20 +1648,35 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
log.Debug("Committed block data", "size", common.StorageSize(batch.ValueSize()), "elapsed", common.PrettyDuration(time.Since(start)))
var (
err error
root common.Hash
isEIP158 = bc.chainConfig.IsEIP158(block.Number())
isCancun = bc.chainConfig.IsCancun(block.Number(), block.Time())
err error
root common.Hash
isEIP158 = bc.chainConfig.IsEIP158(block.Number())
isCancun = bc.chainConfig.IsCancun(block.Number(), block.Time())
hasStateHook = bc.logger != nil && bc.logger.OnStateUpdate != nil
hasStateSizer = bc.stateSizer != nil
)
if bc.stateSizer == nil {
root, err = statedb.Commit(block.NumberU64(), isEIP158, isCancun)
if hasStateHook || hasStateSizer {
r, update, err := statedb.CommitWithUpdate(block.NumberU64(), isEIP158, isCancun)
if err != nil {
return err
}
if hasStateHook {
trUpdate, err := update.ToTracingUpdate()
if err != nil {
return err
}
bc.logger.OnStateUpdate(trUpdate)
}
if hasStateSizer {
bc.stateSizer.Notify(update)
}
root = r
} else {
root, err = statedb.CommitAndTrack(block.NumberU64(), isEIP158, isCancun, bc.stateSizer)
root, err = statedb.Commit(block.NumberU64(), isEIP158, isCancun)
if err != nil {
return err
}
}
if err != nil {
return err
}
// If node is running in path mode, skip explicit gc operation
// which is unnecessary in this mode.
if bc.triedb.Scheme() == rawdb.PathScheme {

10
core/blockchain_reader.go
View file

@ -522,3 +522,13 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript
func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription {
return bc.scope.Track(bc.blockProcFeed.Subscribe(ch))
}
// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent.
func (bc *BlockChain) SubscribeNewPayloadEvent(ch chan<- NewPayloadEvent) event.Subscription {
return bc.scope.Track(bc.newPayloadFeed.Subscribe(ch))
}
// SendNewPayloadEvent sends a NewPayloadEvent to subscribers.
func (bc *BlockChain) SendNewPayloadEvent(ev NewPayloadEvent) {
bc.newPayloadFeed.Send(ev)
}

29
core/blockchain_stats.go
View file

@ -115,26 +115,31 @@ func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Durat
Block: %v (%#x) txs: %d, mgasps: %.2f, elapsed: %v
EVM execution: %v
Validation: %v
Account hash: %v
Storage hash: %v
State read: %v
Account read: %v(%d)
Storage read: %v(%d)
Code read: %v(%d)
State hash: %v
Account hash: %v
Storage hash: %v
State write: %v
Trie commit: %v
DB write: %v
State write: %v
Block write: %v
%s
##############################
`, block.Number(), block.Hash(), len(block.Transactions()), s.MgasPerSecond, common.PrettyDuration(s.TotalTime),
// EVM execution
common.PrettyDuration(s.Execution),
common.PrettyDuration(s.Validation+s.CrossValidation),
// Block validation
common.PrettyDuration(s.Validation+s.CrossValidation+s.AccountHashes+s.AccountUpdates+s.StorageUpdates),
common.PrettyDuration(s.AccountHashes+s.AccountUpdates),
common.PrettyDuration(s.StorageUpdates),
// State read
common.PrettyDuration(s.AccountReads+s.StorageReads+s.CodeReads),
@ -142,19 +147,15 @@ DB write: %v
common.PrettyDuration(s.StorageReads), s.StorageLoaded,
common.PrettyDuration(s.CodeReads), s.CodeLoaded,
// State hash
common.PrettyDuration(s.AccountHashes+s.AccountUpdates+s.StorageUpdates+max(s.AccountCommits, s.StorageCommits)),
common.PrettyDuration(s.AccountHashes+s.AccountUpdates),
common.PrettyDuration(s.StorageUpdates),
// State write
common.PrettyDuration(max(s.AccountCommits, s.StorageCommits)+s.TrieDBCommit+s.SnapshotCommit+s.BlockWrite),
common.PrettyDuration(max(s.AccountCommits, s.StorageCommits)),
// Database commit
common.PrettyDuration(s.TrieDBCommit+s.SnapshotCommit+s.BlockWrite),
common.PrettyDuration(s.TrieDBCommit+s.SnapshotCommit),
common.PrettyDuration(s.BlockWrite),
// cache statistics
s.StateReadCacheStats)
s.StateReadCacheStats,
)
for _, line := range strings.Split(msg, "\n") {
if line == "" {
continue

2
core/chain_makers.go
View file

@ -481,7 +481,7 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int,
}
triedb := triedb.NewDatabase(db, triedbConfig)
defer triedb.Close()
_, err := genesis.Commit(db, triedb)
_, err := genesis.Commit(db, triedb, nil)
if err != nil {
panic(err)
}

10
core/events.go
View file

@ -17,6 +17,9 @@
package core
import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
@ -35,3 +38,10 @@ type ChainEvent struct {
type ChainHeadEvent struct {
Header *types.Header
}
// NewPayloadEvent is posted when engine_newPayloadVX processes a block.
type NewPayloadEvent struct {
Hash common.Hash
Number uint64
ProcessingTime time.Duration
}

38
core/genesis.go
View file

@ -164,7 +164,7 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) {
// flushAlloc is very similar with hash, but the main difference is all the
// generated states will be persisted into the given database.
func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, error) {
func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database, tracer *tracing.Hooks) (common.Hash, error) {
emptyRoot := types.EmptyRootHash
if triedb.IsVerkle() {
emptyRoot = types.EmptyVerkleHash
@ -185,10 +185,26 @@ func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, e
statedb.SetState(addr, key, value)
}
}
root, err := statedb.Commit(0, false, false)
if err != nil {
return common.Hash{}, err
var root common.Hash
if tracer != nil && tracer.OnStateUpdate != nil {
r, update, err := statedb.CommitWithUpdate(0, false, false)
if err != nil {
return common.Hash{}, err
}
trUpdate, err := update.ToTracingUpdate()
if err != nil {
return common.Hash{}, err
}
tracer.OnStateUpdate(trUpdate)
root = r
} else {
root, err = statedb.Commit(0, false, false)
if err != nil {
return common.Hash{}, err
}
}
// Commit newly generated states into disk if it's not empty.
if root != emptyRoot {
if err := triedb.Commit(root, true); err != nil {
@ -296,10 +312,10 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error {
// specify a fork block below the local head block). In case of a conflict, the
// error is a *params.ConfigCompatError and the new, unwritten config is returned.
func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
return SetupGenesisBlockWithOverride(db, triedb, genesis, nil)
return SetupGenesisBlockWithOverride(db, triedb, genesis, nil, nil)
}
func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides, tracer *tracing.Hooks) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
// Copy the genesis, so we can operate on a copy.
genesis = genesis.copy()
// Sanitize the supplied genesis, ensuring it has the associated chain
@ -320,7 +336,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
return nil, common.Hash{}, nil, err
}
block, err := genesis.Commit(db, triedb)
block, err := genesis.Commit(db, triedb, tracer)
if err != nil {
return nil, common.Hash{}, nil, err
}
@ -348,7 +364,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
if hash := genesis.ToBlock().Hash(); hash != ghash {
return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash}
}
block, err := genesis.Commit(db, triedb)
block, err := genesis.Commit(db, triedb, tracer)
if err != nil {
return nil, common.Hash{}, nil, err
}
@ -537,7 +553,7 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block {
// Commit writes the block and state of a genesis specification to the database.
// The block is committed as the canonical head block.
func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) {
func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database, tracer *tracing.Hooks) (*types.Block, error) {
if g.Number != 0 {
return nil, errors.New("can't commit genesis block with number > 0")
}
@ -552,7 +568,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
return nil, errors.New("can't start clique chain without signers")
}
// flush the data to disk and compute the state root
root, err := flushAlloc(&g.Alloc, triedb)
root, err := flushAlloc(&g.Alloc, triedb, tracer)
if err != nil {
return nil, err
}
@ -578,7 +594,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
// MustCommit writes the genesis block and state to db, panicking on error.
// The block is committed as the canonical head block.
func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block {
block, err := g.Commit(db, triedb)
block, err := g.Commit(db, triedb, nil)
if err != nil {
panic(err)
}

10
core/genesis_test.go
View file

@ -88,7 +88,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
name: "custom block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
tdb := triedb.NewDatabase(db, newDbConfig(scheme))
customg.Commit(db, tdb)
customg.Commit(db, tdb, nil)
return SetupGenesisBlock(db, tdb, nil)
},
wantHash: customghash,
@ -98,7 +98,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
name: "custom block in DB, genesis == sepolia",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
tdb := triedb.NewDatabase(db, newDbConfig(scheme))
customg.Commit(db, tdb)
customg.Commit(db, tdb, nil)
return SetupGenesisBlock(db, tdb, DefaultSepoliaGenesisBlock())
},
wantErr: &GenesisMismatchError{Stored: customghash, New: params.SepoliaGenesisHash},
@ -107,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
name: "custom block in DB, genesis == hoodi",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
tdb := triedb.NewDatabase(db, newDbConfig(scheme))
customg.Commit(db, tdb)
customg.Commit(db, tdb, nil)
return SetupGenesisBlock(db, tdb, DefaultHoodiGenesisBlock())
},
wantErr: &GenesisMismatchError{Stored: customghash, New: params.HoodiGenesisHash},
@ -116,7 +116,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
name: "compatible config in DB",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
tdb := triedb.NewDatabase(db, newDbConfig(scheme))
oldcustomg.Commit(db, tdb)
oldcustomg.Commit(db, tdb, nil)
return SetupGenesisBlock(db, tdb, &customg)
},
wantHash: customghash,
@ -128,7 +128,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
// Commit the 'old' genesis block with Homestead transition at #2.
// Advance to block #4, past the homestead transition block of customg.
tdb := triedb.NewDatabase(db, newDbConfig(scheme))
oldcustomg.Commit(db, tdb)
oldcustomg.Commit(db, tdb, nil)
bc, _ := NewBlockChain(db, &oldcustomg, ethash.NewFullFaker(), DefaultConfig().WithStateScheme(scheme))
defer bc.Stop()

2
core/headerchain_test.go
View file

@ -69,7 +69,7 @@ func TestHeaderInsertion(t *testing.T) {
db = rawdb.NewMemoryDatabase()
gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges}
)
gspec.Commit(db, triedb.NewDatabase(db, nil))
gspec.Commit(db, triedb.NewDatabase(db, nil), nil)
hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false })
if err != nil {
t.Fatal(err)

2
core/rawdb/ancient_utils.go
View file

@ -149,6 +149,8 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s
path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs
case MerkleStateFreezerName, VerkleStateFreezerName:
path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs
case MerkleTrienodeFreezerName, VerkleTrienodeFreezerName:
path, tables = filepath.Join(ancient, freezerName), trienodeFreezerTableConfigs
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}

32
core/rawdb/chain_iterator.go
View file

@ -87,6 +87,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
type blockTxHashes struct {
number uint64
hashes []common.Hash
err error
}
// iterateTransactions iterates over all transactions in the (canon) block
@ -144,17 +145,22 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
}()
for data := range rlpCh {
var body types.Body
var result *blockTxHashes
if err := rlp.DecodeBytes(data.rlp, &body); err != nil {
log.Warn("Failed to decode block body", "block", data.number, "error", err)
return
}
var hashes []common.Hash
for _, tx := range body.Transactions {
hashes = append(hashes, tx.Hash())
}
result := &blockTxHashes{
hashes: hashes,
number: data.number,
result = &blockTxHashes{
number: data.number,
err: err,
}
} else {
var hashes []common.Hash
for _, tx := range body.Transactions {
hashes = append(hashes, tx.Hash())
}
result = &blockTxHashes{
hashes: hashes,
number: data.number,
}
}
// Feed the block to the aggregator, or abort on interrupt
select {
@ -214,6 +220,10 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
// Next block available, pop it off and index it
delivery := queue.PopItem()
lastNum = delivery.number
if delivery.err != nil {
log.Warn("Skipping tx indexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err)
continue
}
WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
blocks++
txs += len(delivery.hashes)
@ -307,6 +317,10 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
}
delivery := queue.PopItem()
nextNum = delivery.number + 1
if delivery.err != nil {
log.Warn("Skipping tx unindexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err)
continue
}
DeleteTxLookupEntries(batch, delivery.hashes)
txs += len(delivery.hashes)
blocks++

30
core/rawdb/chain_iterator_test.go
View file

@ -218,6 +218,36 @@ func TestIndexTransactions(t *testing.T) {
verify(0, 8, false, 8)
}
func TestUnindexTransactionsMissingBody(t *testing.T) {
// Construct test chain db
chainDB := NewMemoryDatabase()
blocks, _ := initDatabaseWithTransactions(chainDB)
// Index the entire chain.
lastBlock := blocks[len(blocks)-1].NumberU64()
IndexTransactions(chainDB, 0, lastBlock+1, nil, false)
// Prove that block 2 body exists in the database.
if raw := ReadCanonicalBodyRLP(chainDB, 2, nil); len(raw) == 0 {
t.Fatalf("Block 2 body does not exist in the database.")
}
// Delete body for block 2. This simulates a corrupted database.
key := blockBodyKey(2, blocks[2].Hash())
if err := chainDB.Delete(key); err != nil {
t.Fatalf("Failed to delete block body %v", err)
}
// Unindex blocks [0, 3)
UnindexTransactions(chainDB, 0, 3, nil, false)
// Verify that tx index tail is updated to 3.
tail := ReadTxIndexTail(chainDB)
if tail == nil || *tail != 3 {
t.Fatalf("The tx index tail is wrong: got %v want %d", *tail, 3)
}
}
func TestPruneTransactionIndex(t *testing.T) {
chainDB := NewMemoryDatabase()
blocks, _ := initDatabaseWithTransactions(chainDB)

29
core/state/database.go
View file

@ -177,8 +177,8 @@ func NewDatabaseForTesting() *CachingDB {
return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil)
}
// Reader returns a state reader associated with the specified state root.
func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
// StateReader returns a state reader associated with the specified state root.
func (db *CachingDB) StateReader(stateRoot common.Hash) (StateReader, error) {
var readers []StateReader
// Configure the state reader using the standalone snapshot in hash mode.
@ -208,23 +208,32 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
}
readers = append(readers, tr)
combined, err := newMultiStateReader(readers...)
return newMultiStateReader(readers...)
}
// Reader implements Database, returning a reader associated with the specified
// state root.
func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
sr, err := db.StateReader(stateRoot)
if err != nil {
return nil, err
}
return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil
return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), sr), nil
}
// ReadersWithCacheStats creates a pair of state readers sharing the same internal cache and
// same backing Reader, but exposing separate statistics.
// and statistics.
// ReadersWithCacheStats creates a pair of state readers that share the same
// underlying state reader and internal state cache, while maintaining separate
// statistics respectively.
func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) {
reader, err := db.Reader(stateRoot)
r, err := db.StateReader(stateRoot)
if err != nil {
return nil, nil, err
}
shared := newReaderWithCache(reader)
return newReaderWithCacheStats(shared), newReaderWithCacheStats(shared), nil
sr := newStateReaderWithCache(r)
ra := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache))
rb := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache))
return ra, rb, nil
}
// OpenTrie opens the main account trie at a specific root hash.

84
core/state/reader.go
View file

@ -58,6 +58,13 @@ type ContractCodeReader interface {
CodeSize(addr common.Address, codeHash common.Hash) (int, error)
}
// ContractCodeReaderWithStats extends ContractCodeReader by adding GetStats to
// expose statistics of code reader.
type ContractCodeReaderWithStats interface {
ContractCodeReader
GetStats() (int64, int64)
}
// StateReader defines the interface for accessing accounts and storage slots
// associated with a specific state.
//
@ -97,6 +104,8 @@ type ReaderStats struct {
AccountCacheMiss int64
StorageCacheHit int64
StorageCacheMiss int64
ContractCodeHit int64
ContractCodeMiss int64
}
// String implements fmt.Stringer, returning string format statistics.
@ -104,6 +113,7 @@ func (s ReaderStats) String() string {
var (
accountCacheHitRate float64
storageCacheHitRate float64
contractCodeHitRate float64
)
if s.AccountCacheHit > 0 {
accountCacheHitRate = float64(s.AccountCacheHit) / float64(s.AccountCacheHit+s.AccountCacheMiss) * 100
@ -111,9 +121,13 @@ func (s ReaderStats) String() string {
if s.StorageCacheHit > 0 {
storageCacheHitRate = float64(s.StorageCacheHit) / float64(s.StorageCacheHit+s.StorageCacheMiss) * 100
}
if s.ContractCodeHit > 0 {
contractCodeHitRate = float64(s.ContractCodeHit) / float64(s.ContractCodeHit+s.ContractCodeMiss) * 100
}
msg := fmt.Sprintf("Reader statistics\n")
msg += fmt.Sprintf("account: hit: %d, miss: %d, rate: %.2f\n", s.AccountCacheHit, s.AccountCacheMiss, accountCacheHitRate)
msg += fmt.Sprintf("storage: hit: %d, miss: %d, rate: %.2f\n", s.StorageCacheHit, s.StorageCacheMiss, storageCacheHitRate)
msg += fmt.Sprintf("code: hit: %d, miss: %d, rate: %.2f\n", s.ContractCodeHit, s.ContractCodeMiss, contractCodeHitRate)
return msg
}
@ -134,6 +148,10 @@ type cachingCodeReader struct {
// they are natively thread-safe.
codeCache *lru.SizeConstrainedCache[common.Hash, []byte]
codeSizeCache *lru.Cache[common.Hash, int]
// Cache statistics
hit atomic.Int64 // Number of code lookups found in the cache.
miss atomic.Int64 // Number of code lookups not found in the cache.
}
// newCachingCodeReader constructs the code reader.
@ -150,8 +168,11 @@ func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstraine
func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) {
code, _ := r.codeCache.Get(codeHash)
if len(code) > 0 {
r.hit.Add(1)
return code, nil
}
r.miss.Add(1)
code = rawdb.ReadCode(r.db, codeHash)
if len(code) > 0 {
r.codeCache.Add(codeHash, code)
@ -164,6 +185,7 @@ func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]b
// If the contract code doesn't exist, no error will be returned.
func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) {
if cached, ok := r.codeSizeCache.Get(codeHash); ok {
r.hit.Add(1)
return cached, nil
}
code, err := r.Code(addr, codeHash)
@ -180,6 +202,11 @@ func (r *cachingCodeReader) Has(addr common.Address, codeHash common.Hash) bool
return len(code) > 0
}
// GetStats returns the cache statistics fo the code reader.
func (r *cachingCodeReader) GetStats() (int64, int64) {
return r.hit.Load(), r.miss.Load()
}
// flatReader wraps a database state reader and is safe for concurrent access.
type flatReader struct {
reader database.StateReader
@ -462,10 +489,10 @@ func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader {
}
}
// readerWithCache is a wrapper around Reader that maintains additional state caches
// to support concurrent state access.
type readerWithCache struct {
Reader // safe for concurrent read
// stateReaderWithCache is a wrapper around StateReader that maintains additional
// state caches to support concurrent state access.
type stateReaderWithCache struct {
StateReader
// Previously resolved state entries.
accounts map[common.Address]*types.StateAccount
@ -481,11 +508,11 @@ type readerWithCache struct {
}
}
// newReaderWithCache constructs the reader with local cache.
func newReaderWithCache(reader Reader) *readerWithCache {
r := &readerWithCache{
Reader: reader,
accounts: make(map[common.Address]*types.StateAccount),
// newStateReaderWithCache constructs the state reader with local cache.
func newStateReaderWithCache(sr StateReader) *stateReaderWithCache {
r := &stateReaderWithCache{
StateReader: sr,
accounts: make(map[common.Address]*types.StateAccount),
}
for i := range r.storageBuckets {
r.storageBuckets[i].storages = make(map[common.Address]map[common.Hash]common.Hash)
@ -498,7 +525,7 @@ func newReaderWithCache(reader Reader) *readerWithCache {
// might be nil if it's not existent.
//
// An error will be returned if the state is corrupted in the underlying reader.
func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, bool, error) {
func (r *stateReaderWithCache) account(addr common.Address) (*types.StateAccount, bool, error) {
// Try to resolve the requested account in the local cache
r.accountLock.RLock()
acct, ok := r.accounts[addr]
@ -507,7 +534,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo
return acct, true, nil
}
// Try to resolve the requested account from the underlying reader
acct, err := r.Reader.Account(addr)
acct, err := r.StateReader.Account(addr)
if err != nil {
return nil, false, err
}
@ -521,7 +548,7 @@ func (r *readerWithCache) account(addr common.Address) (*types.StateAccount, boo
// The returned account might be nil if it's not existent.
//
// An error will be returned if the state is corrupted in the underlying reader.
func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, error) {
func (r *stateReaderWithCache) Account(addr common.Address) (*types.StateAccount, error) {
account, _, err := r.account(addr)
return account, err
}
@ -529,7 +556,7 @@ func (r *readerWithCache) Account(addr common.Address) (*types.StateAccount, err
// storage retrieves the storage slot specified by the address and slot key, along
// with a flag indicating whether it's found in the cache or not. The returned
// storage slot might be empty if it's not existent.
func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) {
func (r *stateReaderWithCache) storage(addr common.Address, slot common.Hash) (common.Hash, bool, error) {
var (
value common.Hash
ok bool
@ -546,7 +573,7 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common
return value, true, nil
}
// Try to resolve the requested storage slot from the underlying reader
value, err := r.Reader.Storage(addr, slot)
value, err := r.StateReader.Storage(addr, slot)
if err != nil {
return common.Hash{}, false, err
}
@ -567,13 +594,14 @@ func (r *readerWithCache) storage(addr common.Address, slot common.Hash) (common
// existent.
//
// An error will be returned if the state is corrupted in the underlying reader.
func (r *readerWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
func (r *stateReaderWithCache) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
value, _, err := r.storage(addr, slot)
return value, err
}
type readerWithCacheStats struct {
*readerWithCache
type readerWithStats struct {
*stateReaderWithCache
ContractCodeReaderWithStats
accountCacheHit atomic.Int64
accountCacheMiss atomic.Int64
@ -581,10 +609,11 @@ type readerWithCacheStats struct {
storageCacheMiss atomic.Int64
}
// newReaderWithCacheStats constructs the reader with additional statistics tracked.
func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats {
return &readerWithCacheStats{
readerWithCache: reader,
// newReaderWithStats constructs the reader with additional statistics tracked.
func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats) *readerWithStats {
return &readerWithStats{
stateReaderWithCache: sr,
ContractCodeReaderWithStats: cr,
}
}
@ -592,8 +621,8 @@ func newReaderWithCacheStats(reader *readerWithCache) *readerWithCacheStats {
// The returned account might be nil if it's not existent.
//
// An error will be returned if the state is corrupted in the underlying reader.
func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount, error) {
account, incache, err := r.readerWithCache.account(addr)
func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, error) {
account, incache, err := r.stateReaderWithCache.account(addr)
if err != nil {
return nil, err
}
@ -610,8 +639,8 @@ func (r *readerWithCacheStats) Account(addr common.Address) (*types.StateAccount
// existent.
//
// An error will be returned if the state is corrupted in the underlying reader.
func (r *readerWithCacheStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
value, incache, err := r.readerWithCache.storage(addr, slot)
func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) {
value, incache, err := r.stateReaderWithCache.storage(addr, slot)
if err != nil {
return common.Hash{}, err
}
@ -624,11 +653,14 @@ func (r *readerWithCacheStats) Storage(addr common.Address, slot common.Hash) (c
}
// GetStats implements ReaderWithStats, returning the statistics of state reader.
func (r *readerWithCacheStats) GetStats() ReaderStats {
func (r *readerWithStats) GetStats() ReaderStats {
codeHit, codeMiss := r.ContractCodeReaderWithStats.GetStats()
return ReaderStats{
AccountCacheHit: r.accountCacheHit.Load(),
AccountCacheMiss: r.accountCacheMiss.Load(),
StorageCacheHit: r.storageCacheHit.Load(),
StorageCacheMiss: r.storageCacheMiss.Load(),
ContractCodeHit: codeHit,
ContractCodeMiss: codeMiss,
}
}

6
core/state/state_object.go
View file

@ -440,6 +440,12 @@ func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) {
blob: s.code,
}
s.dirtyCode = false // reset the dirty flag
if s.origin == nil {
op.code.originHash = types.EmptyCodeHash
} else {
op.code.originHash = common.BytesToHash(s.origin.CodeHash)
}
}
// Commit storage changes and the associated storage trie
s.commitStorage(op)

2
core/state/state_sizer.go
View file

@ -245,7 +245,7 @@ func calSizeStats(update *stateUpdate) (SizeStats, error) {
codeExists := make(map[common.Hash]struct{})
for _, code := range update.codes {
if _, ok := codeExists[code.hash]; ok || code.exists {
if _, ok := codeExists[code.hash]; ok || code.duplicate {
continue
}
stats.ContractCodes += 1

54
core/state/statedb.go
View file

@ -509,21 +509,13 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common
}
// SelfDestruct marks the given account as selfdestructed.
// This clears the account balance.
//
// The account's state object is still available until the state is committed,
// getStateObject will return a non-nil account after SelfDestruct.
func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int {
func (s *StateDB) SelfDestruct(addr common.Address) {
stateObject := s.getStateObject(addr)
var prevBalance uint256.Int
if stateObject == nil {
return prevBalance
}
prevBalance = *(stateObject.Balance())
// Regardless of whether it is already destructed or not, we do have to
// journal the balance-change, if we set it to zero here.
if !stateObject.Balance().IsZero() {
stateObject.SetBalance(new(uint256.Int))
return
}
// If it is already marked as self-destructed, we do not need to add it
// for journalling a second time.
@ -531,18 +523,6 @@ func (s *StateDB) SelfDestruct(addr common.Address) uint256.Int {
s.journal.destruct(addr)
stateObject.markSelfdestructed()
}
return prevBalance
}
func (s *StateDB) SelfDestruct6780(addr common.Address) (uint256.Int, bool) {
stateObject := s.getStateObject(addr)
if stateObject == nil {
return uint256.Int{}, false
}
if stateObject.newContract {
return s.SelfDestruct(addr), true
}
return *(stateObject.Balance()), false
}
// SetTransientState sets transient storage for a given account. It
@ -670,6 +650,16 @@ func (s *StateDB) CreateContract(addr common.Address) {
}
}
// IsNewContract reports whether the contract at the given address was deployed
// during the current transaction.
func (s *StateDB) IsNewContract(addr common.Address) bool {
obj := s.getStateObject(addr)
if obj == nil {
return false
}
return obj.newContract
}
// Copy creates a deep, independent copy of the state.
// Snapshots of the copied state cannot be applied to the copy.
func (s *StateDB) Copy() *StateDB {
@ -1318,16 +1308,16 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum
// commitAndFlush is a wrapper of commit which also commits the state mutations
// to the configured data stores.
func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool, dedupCode bool) (*stateUpdate, error) {
func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorageWiping bool, deriveCodeFields bool) (*stateUpdate, error) {
ret, err := s.commit(deleteEmptyObjects, noStorageWiping, block)
if err != nil {
return nil, err
}
if dedupCode {
ret.markCodeExistence(s.reader)
if deriveCodeFields {
if err := ret.deriveCodeFields(s.reader); err != nil {
return nil, err
}
}
// Commit dirty contract code if any exists
if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 {
batch := db.NewBatch()
@ -1389,14 +1379,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool, noStorageWiping
return ret.root, nil
}
// CommitAndTrack writes the state mutations and notifies the size tracker of the state changes.
func (s *StateDB) CommitAndTrack(block uint64, deleteEmptyObjects bool, noStorageWiping bool, sizer *SizeTracker) (common.Hash, error) {
// CommitWithUpdate writes the state mutations and returns the state update for
// external processing (e.g., live tracing hooks or size tracker).
func (s *StateDB) CommitWithUpdate(block uint64, deleteEmptyObjects bool, noStorageWiping bool) (common.Hash, *stateUpdate, error) {
ret, err := s.commitAndFlush(block, deleteEmptyObjects, noStorageWiping, true)
if err != nil {
return common.Hash{}, err
return common.Hash{}, nil, err
}
sizer.Notify(ret)
return ret.root, nil
return ret.root, ret, nil
}
// Prepare handles the preparatory steps for executing a state transition with.

107
core/state/statedb_hooked.go
View file

@ -17,7 +17,9 @@
package state
import (
"bytes"
"math/big"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/stateless"
@ -52,6 +54,10 @@ func (s *hookedStateDB) CreateContract(addr common.Address) {
s.inner.CreateContract(addr)
}
func (s *hookedStateDB) IsNewContract(addr common.Address) bool {
return s.inner.IsNewContract(addr)
}
func (s *hookedStateDB) GetBalance(addr common.Address) *uint256.Int {
return s.inner.GetBalance(addr)
}
@ -211,56 +217,8 @@ func (s *hookedStateDB) SetState(address common.Address, key common.Hash, value
return prev
}
func (s *hookedStateDB) SelfDestruct(address common.Address) uint256.Int {
var prevCode []byte
var prevCodeHash common.Hash
if s.hooks.OnCodeChange != nil || s.hooks.OnCodeChangeV2 != nil {
prevCode = s.inner.GetCode(address)
prevCodeHash = s.inner.GetCodeHash(address)
}
prev := s.inner.SelfDestruct(address)
if s.hooks.OnBalanceChange != nil && !prev.IsZero() {
s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct)
}
if len(prevCode) > 0 {
if s.hooks.OnCodeChangeV2 != nil {
s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct)
} else if s.hooks.OnCodeChange != nil {
s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil)
}
}
return prev
}
func (s *hookedStateDB) SelfDestruct6780(address common.Address) (uint256.Int, bool) {
var prevCode []byte
var prevCodeHash common.Hash
if s.hooks.OnCodeChange != nil || s.hooks.OnCodeChangeV2 != nil {
prevCodeHash = s.inner.GetCodeHash(address)
prevCode = s.inner.GetCode(address)
}
prev, changed := s.inner.SelfDestruct6780(address)
if s.hooks.OnBalanceChange != nil && !prev.IsZero() {
s.hooks.OnBalanceChange(address, prev.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestruct)
}
if changed && len(prevCode) > 0 {
if s.hooks.OnCodeChangeV2 != nil {
s.hooks.OnCodeChangeV2(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct)
} else if s.hooks.OnCodeChange != nil {
s.hooks.OnCodeChange(address, prevCodeHash, prevCode, types.EmptyCodeHash, nil)
}
}
return prev, changed
func (s *hookedStateDB) SelfDestruct(address common.Address) {
s.inner.SelfDestruct(address)
}
func (s *hookedStateDB) AddLog(log *types.Log) {
@ -272,17 +230,58 @@ func (s *hookedStateDB) AddLog(log *types.Log) {
}
func (s *hookedStateDB) Finalise(deleteEmptyObjects bool) {
defer s.inner.Finalise(deleteEmptyObjects)
if s.hooks.OnBalanceChange == nil {
if s.hooks.OnBalanceChange == nil && s.hooks.OnNonceChangeV2 == nil && s.hooks.OnNonceChange == nil && s.hooks.OnCodeChangeV2 == nil && s.hooks.OnCodeChange == nil {
// Short circuit if no relevant hooks are set.
s.inner.Finalise(deleteEmptyObjects)
return
}
// Collect all self-destructed addresses first, then sort them to ensure
// that state change hooks will be invoked in deterministic
// order when the accounts are deleted below
var selfDestructedAddrs []common.Address
for addr := range s.inner.journal.dirties {
obj := s.inner.stateObjects[addr]
if obj != nil && obj.selfDestructed {
// If ether was sent to account post-selfdestruct it is burnt.
if obj == nil || !obj.selfDestructed {
// Not self-destructed, keep searching.
continue
}
selfDestructedAddrs = append(selfDestructedAddrs, addr)
}
sort.Slice(selfDestructedAddrs, func(i, j int) bool {
return bytes.Compare(selfDestructedAddrs[i][:], selfDestructedAddrs[j][:]) < 0
})
for _, addr := range selfDestructedAddrs {
obj := s.inner.stateObjects[addr]
// Bingo: state object was self-destructed, call relevant hooks.
// If ether was sent to account post-selfdestruct, record as burnt.
if s.hooks.OnBalanceChange != nil {
if bal := obj.Balance(); bal.Sign() != 0 {
s.hooks.OnBalanceChange(addr, bal.ToBig(), new(big.Int), tracing.BalanceDecreaseSelfdestructBurn)
}
}
// Nonce is set to reset on self-destruct.
if s.hooks.OnNonceChangeV2 != nil {
s.hooks.OnNonceChangeV2(addr, obj.Nonce(), 0, tracing.NonceChangeSelfdestruct)
} else if s.hooks.OnNonceChange != nil {
s.hooks.OnNonceChange(addr, obj.Nonce(), 0)
}
// If an initcode invokes selfdestruct, do not emit a code change.
prevCodeHash := s.inner.GetCodeHash(addr)
if prevCodeHash == types.EmptyCodeHash {
continue
}
// Otherwise, trace the change.
if s.hooks.OnCodeChangeV2 != nil {
s.hooks.OnCodeChangeV2(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil, tracing.CodeChangeSelfDestruct)
} else if s.hooks.OnCodeChange != nil {
s.hooks.OnCodeChange(addr, prevCodeHash, s.inner.GetCode(addr), types.EmptyCodeHash, nil)
}
}
s.inner.Finalise(deleteEmptyObjects)
}

7
core/state/statedb_hooked_test.go
View file

@ -49,6 +49,8 @@ func TestBurn(t *testing.T) {
createAndDestroy := func(addr common.Address) {
hooked.AddBalance(addr, uint256.NewInt(100), tracing.BalanceChangeUnspecified)
hooked.CreateContract(addr)
// Simulate what the opcode handler does: clear balance before selfdestruct
hooked.SubBalance(addr, hooked.GetBalance(addr), tracing.BalanceDecreaseSelfdestruct)
hooked.SelfDestruct(addr)
// sanity-check that balance is now 0
if have, want := hooked.GetBalance(addr), new(uint256.Int); !have.Eq(want) {
@ -140,8 +142,8 @@ func TestHooks_OnCodeChangeV2(t *testing.T) {
var result []string
var wants = []string{
"0xaa00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) ContractCreation",
"0xaa00000000000000000000000000000000000000.code: 0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct",
"0xbb00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) ContractCreation",
"0xaa00000000000000000000000000000000000000.code: 0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct",
"0xbb00000000000000000000000000000000000000.code: 0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct",
}
emitF := func(format string, a ...any) {
@ -157,7 +159,8 @@ func TestHooks_OnCodeChangeV2(t *testing.T) {
sdb.SetCode(common.Address{0xbb}, []byte{0x13, 38}, tracing.CodeChangeContractCreation)
sdb.CreateContract(common.Address{0xbb})
sdb.SelfDestruct6780(common.Address{0xbb})
sdb.SelfDestruct(common.Address{0xbb})
sdb.Finalise(true)
if len(result) != len(wants) {
t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants))

181
core/state/stateupdate.go
View file

@ -17,18 +17,27 @@
package state
import (
"fmt"
"maps"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/tracing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/triedb"
)
// contractCode represents a contract code with associated metadata.
// contractCode represents contract bytecode along with its associated metadata.
type contractCode struct {
hash common.Hash // hash is the cryptographic hash of the contract code.
blob []byte // blob is the binary representation of the contract code.
exists bool // flag whether the code has been existent
hash common.Hash // hash is the cryptographic hash of the current contract code.
blob []byte // blob is the binary representation of the current contract code.
originHash common.Hash // originHash is the cryptographic hash of the code before mutation.
// Derived fields, populated only when state tracking is enabled.
duplicate bool // duplicate indicates whether the updated code already exists.
originBlob []byte // originBlob is the original binary representation of the contract code.
}
// accountDelete represents an operation for deleting an Ethereum account.
@ -192,21 +201,169 @@ func (sc *stateUpdate) stateSet() *triedb.StateSet {
}
}
// markCodeExistence determines whether each piece of contract code referenced
// in this state update actually exists.
// deriveCodeFields derives the missing fields of contract code changes
// such as original code value.
//
// Note: This operation is expensive and not needed during normal state transitions.
// It is only required when SizeTracker is enabled to produce accurate state
// statistics.
func (sc *stateUpdate) markCodeExistence(reader ContractCodeReader) {
// Note: This operation is expensive and not needed during normal state
// transitions. It is only required when SizeTracker or StateUpdate hook
// is enabled to produce accurate state statistics.
func (sc *stateUpdate) deriveCodeFields(reader ContractCodeReader) error {
cache := make(map[common.Hash]bool)
for addr, code := range sc.codes {
if code.originHash != types.EmptyCodeHash {
blob, err := reader.Code(addr, code.originHash)
if err != nil {
return err
}
code.originBlob = blob
}
if exists, ok := cache[code.hash]; ok {
code.exists = exists
code.duplicate = exists
continue
}
res := reader.Has(addr, code.hash)
cache[code.hash] = res
code.exists = res
code.duplicate = res
}
return nil
}
// ToTracingUpdate converts the internal stateUpdate to an exported tracing.StateUpdate.
func (sc *stateUpdate) ToTracingUpdate() (*tracing.StateUpdate, error) {
update := &tracing.StateUpdate{
OriginRoot: sc.originRoot,
Root: sc.root,
BlockNumber: sc.blockNumber,
AccountChanges: make(map[common.Address]*tracing.AccountChange, len(sc.accountsOrigin)),
StorageChanges: make(map[common.Address]map[common.Hash]*tracing.StorageChange),
CodeChanges: make(map[common.Address]*tracing.CodeChange, len(sc.codes)),
TrieChanges: make(map[common.Hash]map[string]*tracing.TrieNodeChange),
}
// Gather all account changes
for addr, oldData := range sc.accountsOrigin {
addrHash := crypto.Keccak256Hash(addr.Bytes())
newData, exists := sc.accounts[addrHash]
if !exists {
return nil, fmt.Errorf("account %x not found", addr)
}
change := &tracing.AccountChange{}
if len(oldData) > 0 {
acct, err := types.FullAccount(oldData)
if err != nil {
return nil, err
}
change.Prev = &types.StateAccount{
Nonce: acct.Nonce,
Balance: acct.Balance,
Root: acct.Root,
CodeHash: acct.CodeHash,
}
}
if len(newData) > 0 {
acct, err := types.FullAccount(newData)
if err != nil {
return nil, err
}
change.New = &types.StateAccount{
Nonce: acct.Nonce,
Balance: acct.Balance,
Root: acct.Root,
CodeHash: acct.CodeHash,
}
}
update.AccountChanges[addr] = change
}
// Gather all storage slot changes
for addr, slots := range sc.storagesOrigin {
addrHash := crypto.Keccak256Hash(addr.Bytes())
subset, exists := sc.storages[addrHash]
if !exists {
return nil, fmt.Errorf("storage %x not found", addr)
}
storageChanges := make(map[common.Hash]*tracing.StorageChange, len(slots))
for key, encPrev := range slots {
// Get new value - handle both raw and hashed key formats
var (
exists bool
encNew []byte
decPrev []byte
decNew []byte
err error
)
if sc.rawStorageKey {
encNew, exists = subset[crypto.Keccak256Hash(key.Bytes())]
} else {
encNew, exists = subset[key]
}
if !exists {
return nil, fmt.Errorf("storage slot %x-%x not found", addr, key)
}
// Decode the prev and new values
if len(encPrev) > 0 {
_, decPrev, _, err = rlp.Split(encPrev)
if err != nil {
return nil, fmt.Errorf("failed to decode prevValue: %v", err)
}
}
if len(encNew) > 0 {
_, decNew, _, err = rlp.Split(encNew)
if err != nil {
return nil, fmt.Errorf("failed to decode newValue: %v", err)
}
}
storageChanges[key] = &tracing.StorageChange{
Prev: common.BytesToHash(decPrev),
New: common.BytesToHash(decNew),
}
}
update.StorageChanges[addr] = storageChanges
}
// Gather all contract code changes
for addr, code := range sc.codes {
change := &tracing.CodeChange{
New: &tracing.ContractCode{
Hash: code.hash,
Code: code.blob,
Exists: code.duplicate,
},
}
if code.originHash != types.EmptyCodeHash {
change.Prev = &tracing.ContractCode{
Hash: code.originHash,
Code: code.originBlob,
Exists: true,
}
}
update.CodeChanges[addr] = change
}
// Gather all trie node changes
if sc.nodes != nil {
for owner, subset := range sc.nodes.Sets {
nodeChanges := make(map[string]*tracing.TrieNodeChange, len(subset.Origins))
for path, oldNode := range subset.Origins {
newNode, exists := subset.Nodes[path]
if !exists {
return nil, fmt.Errorf("node %x-%v not found", owner, path)
}
nodeChanges[path] = &tracing.TrieNodeChange{
Prev: &trienode.Node{
Hash: crypto.Keccak256Hash(oldNode),
Blob: oldNode,
},
New: &trienode.Node{
Hash: newNode.Hash,
Blob: newNode.Blob,
},
}
}
update.TrieChanges[owner] = nodeChanges
}
}
return update, nil
}

5
core/tracing/gen_nonce_change_reason_stringer.go
View file

@ -15,11 +15,12 @@ func _() {
_ = x[NonceChangeNewContract-4]
_ = x[NonceChangeAuthorization-5]
_ = x[NonceChangeRevert-6]
_ = x[NonceChangeSelfdestruct-7]
}
const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevert"
const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewContractAuthorizationRevertSelfdestruct"
var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70}
var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70, 82}
func (i NonceChangeReason) String() string {
if i >= NonceChangeReason(len(_NonceChangeReason_index)-1) {

60
core/tracing/hooks.go
View file

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/holiman/uint256"
)
@ -75,6 +76,56 @@ type BlockEvent struct {
Safe *types.Header
}
// StateUpdate represents the state mutations resulting from block execution.
// It provides access to account changes, storage changes, and contract code
// deployments with both previous and new values.
type StateUpdate struct {
OriginRoot common.Hash // State root before the update
Root common.Hash // State root after the update
BlockNumber uint64
// AccountChanges contains all account state changes keyed by address.
AccountChanges map[common.Address]*AccountChange
// StorageChanges contains all storage slot changes keyed by address and storage slot key.
StorageChanges map[common.Address]map[common.Hash]*StorageChange
// CodeChanges contains all contract code changes keyed by address.
CodeChanges map[common.Address]*CodeChange
// TrieChanges contains trie node mutations keyed by address hash and trie node path.
TrieChanges map[common.Hash]map[string]*TrieNodeChange
}
// AccountChange represents a change to an account's state.
type AccountChange struct {
Prev *types.StateAccount // nil if account was created
New *types.StateAccount // nil if account was deleted
}
// StorageChange represents a change to a storage slot.
type StorageChange struct {
Prev common.Hash // previous value (zero if slot was created)
New common.Hash // new value (zero if slot was deleted)
}
type ContractCode struct {
Hash common.Hash
Code []byte
Exists bool // true if the code was existent
}
// CodeChange represents a change in contract code of an account.
type CodeChange struct {
Prev *ContractCode // nil if no code existed before
New *ContractCode
}
type TrieNodeChange struct {
Prev *trienode.Node
New *trienode.Node
}
type (
/*
- VM events -
@ -161,6 +212,11 @@ type (
// beacon block root.
OnSystemCallEndHook = func()
// StateUpdateHook is called after state is committed for a block.
// It provides access to the complete state mutations including account changes,
// storage changes, trie node mutations, and contract code deployments.
StateUpdateHook = func(update *StateUpdate)
/*
- State events -
*/
@ -209,6 +265,7 @@ type Hooks struct {
OnSystemCallStart OnSystemCallStartHook
OnSystemCallStartV2 OnSystemCallStartHookV2
OnSystemCallEnd OnSystemCallEndHook
OnStateUpdate StateUpdateHook
// State events
OnBalanceChange BalanceChangeHook
OnNonceChange NonceChangeHook
@ -375,6 +432,9 @@ const (
// NonceChangeRevert is emitted when the nonce is reverted back to a previous value due to call failure.
// It is only emitted when the tracer has opted in to use the journaling wrapper (WrapWithJournal).
NonceChangeRevert NonceChangeReason = 6
// NonceChangeSelfdestruct is emitted when the nonce is reset to zero due to a self-destruct
NonceChangeSelfdestruct NonceChangeReason = 7
)
// CodeChangeReason is used to indicate the reason for a code change.

156
core/txpool/blobpool/blobpool.go
View file

@ -94,6 +94,16 @@ const (
// storeVersion is the current slotter layout used for the billy.Database
// store.
storeVersion = 1
// gappedLifetime is the approximate duration for which nonce-gapped transactions
// are kept before being dropped. Since gapped is only a reorder buffer and it
// is expected that the original transactions were inserted in the mempool in
// nonce order, the duration is kept short to avoid DoS vectors.
gappedLifetime = 1 * time.Minute
// maxGappedTxs is the maximum number of gapped transactions kept overall.
// This is a safety limit to avoid DoS vectors.
maxGapped = 128
)
// blobTxMeta is the minimal subset of types.BlobTx necessary to validate and
@ -330,6 +340,9 @@ type BlobPool struct {
stored uint64 // Useful data size of all transactions on disk
limbo *limbo // Persistent data store for the non-finalized blobs
gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
signer types.Signer // Transaction signer to use for sender recovery
chain BlockChain // Chain object to access the state through
@ -363,6 +376,8 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo
lookup: newLookup(),
index: make(map[common.Address][]*blobTxMeta),
spent: make(map[common.Address]*uint256.Int),
gapped: make(map[common.Address][]*types.Transaction),
gappedSource: make(map[common.Hash]common.Address),
}
}
@ -834,6 +849,9 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
resettimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
// Handle reorg buffer timeouts evicting old gapped transactions
p.evictGapped()
statedb, err := p.chain.StateAt(newHead.Root)
if err != nil {
log.Error("Failed to reset blobpool state", "err", err)
@ -1196,7 +1214,9 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
State: p.state,
FirstNonceGap: func(addr common.Address) uint64 {
// Nonce gaps are not permitted in the blob pool, the first gap will
// Nonce gaps are permitted in the blob pool, but only as part of the
// in-memory 'gapped' buffer. We expose the gap here to validateTx,
// then handle the error by adding to the buffer. The first gap will
// be the next nonce shifted by however many transactions we already
// have pooled.
return p.state.GetNonce(addr) + uint64(len(p.index[addr]))
@ -1275,7 +1295,9 @@ func (p *BlobPool) Has(hash common.Hash) bool {
p.lock.RLock()
defer p.lock.RUnlock()
return p.lookup.exists(hash)
poolHas := p.lookup.exists(hash)
_, gapped := p.gappedSource[hash]
return poolHas || gapped
}
func (p *BlobPool) getRLP(hash common.Hash) []byte {
@ -1466,10 +1488,6 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error {
adds = append(adds, tx.WithoutBlobTxSidecar())
}
}
if len(adds) > 0 {
p.discoverFeed.Send(core.NewTxsEvent{Txs: adds})
p.insertFeed.Send(core.NewTxsEvent{Txs: adds})
}
return errs
}
@ -1488,6 +1506,13 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
return p.addLocked(tx, true)
}
// addLocked inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). It must be called with the pool lock held.
// Only for internal use.
func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) {
// Ensure the transaction is valid from all perspectives
if err := p.validateTx(tx); err != nil {
log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
@ -1500,6 +1525,21 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addStaleMeter.Mark(1)
case errors.Is(err, core.ErrNonceTooHigh):
addGappedMeter.Mark(1)
// Store the tx in memory, and revalidate later
from, _ := types.Sender(p.signer, tx)
allowance := p.gappedAllowance(from)
if allowance >= 1 && len(p.gapped) < maxGapped {
p.gapped[from] = append(p.gapped[from], tx)
p.gappedSource[tx.Hash()] = from
log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
return nil
} else {
// if maxGapped is reached, it is better to give time to gapped
// transactions by keeping the old and dropping this one.
// Thus replacing a gapped transaction with another gapped transaction
// is discouraged.
log.Trace("no gapped blob queue allowance", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
}
case errors.Is(err, core.ErrInsufficientFunds):
addOverdraftedMeter.Mark(1)
case errors.Is(err, txpool.ErrAccountLimitExceeded):
@ -1637,6 +1677,58 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
p.updateStorageMetrics()
addValidMeter.Mark(1)
// Notify all listeners of the new arrival
p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}})
p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}})
//check the gapped queue for this account and try to promote
if gtxs, ok := p.gapped[from]; checkGapped && ok && len(gtxs) > 0 {
// We have to add in nonce order, but we want to stable sort to cater for situations
// where transactions are replaced, keeping the original receive order for same nonce
sort.SliceStable(gtxs, func(i, j int) bool {
return gtxs[i].Nonce() < gtxs[j].Nonce()
})
for len(gtxs) > 0 {
stateNonce := p.state.GetNonce(from)
firstgap := stateNonce + uint64(len(p.index[from]))
if gtxs[0].Nonce() > firstgap {
// Anything beyond the first gap is not addable yet
break
}
// Drop any buffered transactions that became stale in the meantime (included in chain or replaced)
// If we arrive to the transaction in the pending range (between the state Nonce and first gap, we
// try to add them now while removing from here.
tx := gtxs[0]
gtxs[0] = nil
gtxs = gtxs[1:]
delete(p.gappedSource, tx.Hash())
if tx.Nonce() < stateNonce {
// Stale, drop it. Eventually we could add to limbo here if hash matches.
log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
continue
}
if tx.Nonce() <= firstgap {
// If we hit the pending range, including the first gap, add it and continue to try to add more.
// We do not recurse here, but continue to loop instead.
// We are under lock, so we can add the transaction directly.
if err := p.addLocked(tx, false); err == nil {
log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
} else {
log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err)
}
}
}
if len(gtxs) == 0 {
delete(p.gapped, from)
} else {
p.gapped[from] = gtxs
}
}
return nil
}
@ -1868,6 +1960,50 @@ func (p *BlobPool) Nonce(addr common.Address) uint64 {
return p.state.GetNonce(addr)
}
// gappedAllowance returns the number of gapped transactions still
// allowed for the given account. Allowance is based on a slow-start
// logic, allowing more gaps (resource usage) to accounts with a
// higher nonce. Can also return negative values.
func (p *BlobPool) gappedAllowance(addr common.Address) int {
// Gaps happen, but we don't want to allow too many.
// Use log10(nonce+1) as the allowance, with a minimum of 0.
nonce := p.state.GetNonce(addr)
allowance := int(math.Log10(float64(nonce + 1)))
// Cap the allowance to the remaining pool space
return min(allowance, maxTxsPerAccount-len(p.index[addr])) - len(p.gapped[addr])
}
// evictGapped removes the old transactions from the gapped reorder buffer.
// Concurrency: The caller must hold the pool lock before calling this function.
func (p *BlobPool) evictGapped() {
cutoff := time.Now().Add(-gappedLifetime)
for from, txs := range p.gapped {
nonce := p.state.GetNonce(from)
// Reuse the original slice to avoid extra allocations.
// This is safe because we only keep references to the original gappedTx objects,
// and we overwrite the slice for this account after filtering.
keep := txs[:0]
for i, gtx := range txs {
if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce {
// Evict old or stale transactions
// Should we add stale to limbo here if it would belong?
delete(p.gappedSource, gtx.Hash())
txs[i] = nil // Explicitly nil out evicted element
} else {
keep = append(keep, gtx)
}
}
if len(keep) < len(txs) {
log.Trace("Evicting old gapped blob transactions", "count", len(txs)-len(keep), "from", from)
}
if len(keep) == 0 {
delete(p.gapped, from)
} else {
p.gapped[from] = keep
}
}
}
// Stats retrieves the current pool stats, namely the number of pending and the
// number of queued (non-executable) transactions.
func (p *BlobPool) Stats() (int, int) {
@ -1902,9 +2038,15 @@ func (p *BlobPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*ty
// Status returns the known status (unknown/pending/queued) of a transaction
// identified by their hashes.
func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus {
if p.Has(hash) {
p.lock.RLock()
defer p.lock.RUnlock()
if p.lookup.exists(hash) {
return txpool.TxStatusPending
}
if _, gapped := p.gappedSource[hash]; gapped {
return txpool.TxStatusQueued
}
return txpool.TxStatusUnknown
}

41
core/txpool/blobpool/blobpool_test.go
View file

@ -1352,9 +1352,10 @@ func TestAdd(t *testing.T) {
}
// addtx is a helper sender/tx tuple to represent a new tx addition
type addtx struct {
from string
tx *types.BlobTx
err error
from string
tx *types.BlobTx
err error
check func(*BlobPool, *types.Transaction) bool
}
tests := []struct {
@ -1371,6 +1372,7 @@ func TestAdd(t *testing.T) {
"bob": {balance: 21100 + blobSize, nonce: 1},
"claire": {balance: 21100 + blobSize},
"dave": {balance: 21100 + blobSize, nonce: 1},
"eve": {balance: 21100 + blobSize, nonce: 10}, // High nonce to test gapped acceptance
},
adds: []addtx{
{ // New account, no previous txs: accept nonce 0
@ -1398,6 +1400,14 @@ func TestAdd(t *testing.T) {
tx: makeUnsignedTx(2, 1, 1, 1),
err: core.ErrNonceTooHigh,
},
{ // Old account, 10 txs in chain: 0 pending: accept nonce 11 as gapped
from: "eve",
tx: makeUnsignedTx(11, 1, 1, 1),
err: nil,
check: func(pool *BlobPool, tx *types.Transaction) bool {
return pool.Status(tx.Hash()) == txpool.TxStatusQueued
},
},
},
},
// Transactions from already pooled accounts should only be accepted if
@ -1758,15 +1768,28 @@ func TestAdd(t *testing.T) {
t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, errs[0], add.err)
}
if add.err == nil {
size, exist := pool.lookup.sizeOfTx(signed.Hash())
if !exist {
t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j)
// first check if tx is in the pool (reorder queue or pending)
if !pool.Has(signed.Hash()) {
t.Errorf("test %d, tx %d: added transaction not found in pool", i, j)
}
if size != signed.Size() {
t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v",
i, j, size, signed.Size())
// if it is pending, check if size matches
if pool.Status(signed.Hash()) == txpool.TxStatusPending {
size, exist := pool.lookup.sizeOfTx(signed.Hash())
if !exist {
t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j)
}
if size != signed.Size() {
t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v",
i, j, size, signed.Size())
}
}
}
if add.check != nil {
if !add.check(pool, signed) {
t.Errorf("test %d, tx %d: custom check failed", i, j)
}
}
// Verify the pool internals after each addition
verifyPoolInternals(t, pool)
}
verifyPoolInternals(t, pool)

3
core/txpool/errors.go
View file

@ -71,4 +71,7 @@ var (
// ErrInflightTxLimitReached is returned when the maximum number of in-flight
// transactions is reached for specific accounts.
ErrInflightTxLimitReached = errors.New("in-flight transaction limit reached for delegated accounts")
// ErrKZGVerificationError is returned when a KZG proof was not verified correctly.
ErrKZGVerificationError = errors.New("KZG verification error")
)

7
core/txpool/validation.go
View file

@ -202,7 +202,7 @@ func validateBlobSidecarLegacy(sidecar *types.BlobTxSidecar, hashes []common.Has
}
for i := range sidecar.Blobs {
if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("invalid blob %d: %v", i, err)
return fmt.Errorf("%w: invalid blob proof: %v", ErrKZGVerificationError, err)
}
}
return nil
@ -212,7 +212,10 @@ func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash
if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
}
return kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs)
if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
return nil
}
// ValidationOptionsWithState define certain differences between stateful transaction

6
core/vm/contracts.go
View file

@ -299,11 +299,11 @@ func (c *ecrecover) Run(input []byte) ([]byte, error) {
}
// We must make sure not to modify the 'input', so placing the 'v' along with
// the signature needs to be done on a new allocation
sig := make([]byte, 65)
copy(sig, input[64:128])
var sig [65]byte
copy(sig[:], input[64:128])
sig[64] = v
// v needs to be at the end for libsecp256k1
pubKey, err := crypto.Ecrecover(input[:32], sig)
pubKey, err := crypto.Ecrecover(input[:32], sig[:])
// make sure the public key is a valid one
if err != nil {
return nil, nil

14
core/vm/gas_table.go
View file

@ -97,6 +97,9 @@ var (
)
func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
if evm.readOnly {
return 0, ErrWriteProtection
}
var (
y, x = stack.Back(1), stack.Back(0)
current, original = evm.StateDB.GetStateAndCommittedState(contract.Address(), x.Bytes32())
@ -181,6 +184,9 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi
// (2.2.2.1.) If original value is 0, add SSTORE_SET_GAS - SLOAD_GAS to refund counter.
// (2.2.2.2.) Otherwise, add SSTORE_RESET_GAS - SLOAD_GAS gas to refund counter.
func gasSStoreEIP2200(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
if evm.readOnly {
return 0, ErrWriteProtection
}
// If we fail the minimum gas availability invariant, fail (0)
if contract.Gas <= params.SstoreSentryGasEIP2200 {
return 0, errors.New("not enough gas for reentrancy sentry")
@ -374,6 +380,10 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize
transfersValue = !stack.Back(2).IsZero()
address = common.Address(stack.Back(1).Bytes20())
)
if evm.readOnly && transfersValue {
return 0, ErrWriteProtection
}
if evm.chainRules.IsEIP158 {
if transfersValue && evm.StateDB.Empty(address) {
gas += params.CallNewAccountGas
@ -462,6 +472,10 @@ func gasStaticCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memo
}
func gasSelfdestruct(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
if evm.readOnly {
return 0, ErrWriteProtection
}
var gas uint64
// EIP150 homestead gas reprice fork:
if evm.chainRules.IsEIP150 {

50
core/vm/instructions.go
View file

@ -885,13 +885,24 @@ func opSelfdestruct(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
if evm.readOnly {
return nil, ErrWriteProtection
}
beneficiary := scope.Stack.pop()
balance := evm.StateDB.GetBalance(scope.Contract.Address())
evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct)
evm.StateDB.SelfDestruct(scope.Contract.Address())
var (
this = scope.Contract.Address()
balance = evm.StateDB.GetBalance(this)
top = scope.Stack.pop()
beneficiary = common.Address(top.Bytes20())
)
// The funds are burned immediately if the beneficiary is the caller itself,
// in this case, the beneficiary's balance is not increased.
if this != beneficiary {
evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct)
}
// Clear any leftover funds for the account being destructed.
evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct)
evm.StateDB.SelfDestruct(this)
if tracer := evm.Config.Tracer; tracer != nil {
if tracer.OnEnter != nil {
tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig())
tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig())
}
if tracer.OnExit != nil {
tracer.OnExit(evm.depth, []byte{}, 0, nil, false)
@ -904,14 +915,31 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro
if evm.readOnly {
return nil, ErrWriteProtection
}
beneficiary := scope.Stack.pop()
balance := evm.StateDB.GetBalance(scope.Contract.Address())
evm.StateDB.SubBalance(scope.Contract.Address(), balance, tracing.BalanceDecreaseSelfdestruct)
evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, tracing.BalanceIncreaseSelfdestruct)
evm.StateDB.SelfDestruct6780(scope.Contract.Address())
var (
this = scope.Contract.Address()
balance = evm.StateDB.GetBalance(this)
top = scope.Stack.pop()
beneficiary = common.Address(top.Bytes20())
newContract = evm.StateDB.IsNewContract(this)
)
// Contract is new and will actually be deleted.
if newContract {
if this != beneficiary { // Skip no-op transfer when self-destructing to self.
evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct)
}
evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct)
evm.StateDB.SelfDestruct(this)
}
// Contract already exists, only do transfer if beneficiary is not self.
if !newContract && this != beneficiary {
evm.StateDB.SubBalance(this, balance, tracing.BalanceDecreaseSelfdestruct)
evm.StateDB.AddBalance(beneficiary, balance, tracing.BalanceIncreaseSelfdestruct)
}
if tracer := evm.Config.Tracer; tracer != nil {
if tracer.OnEnter != nil {
tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance.ToBig())
tracer.OnEnter(evm.depth, byte(SELFDESTRUCT), this, beneficiary, []byte{}, 0, balance.ToBig())
}
if tracer.OnExit != nil {
tracer.OnExit(evm.depth, []byte{}, 0, nil, false)

14
core/vm/interface.go
View file

@ -57,19 +57,17 @@ type StateDB interface {
GetTransientState(addr common.Address, key common.Hash) common.Hash
SetTransientState(addr common.Address, key, value common.Hash)
SelfDestruct(common.Address) uint256.Int
SelfDestruct(common.Address)
HasSelfDestructed(common.Address) bool
// SelfDestruct6780 is post-EIP6780 selfdestruct, which means that it's a
// send-all-to-beneficiary, unless the contract was created in this same
// transaction, in which case it will be destructed.
// This method returns the prior balance, along with a boolean which is
// true iff the object was indeed destructed.
SelfDestruct6780(common.Address) (uint256.Int, bool)
// Exist reports whether the given account exists in state.
// Notably this also returns true for self-destructed accounts within the current transaction.
Exist(common.Address) bool
// IsNewContract reports whether the contract at the given address was deployed
// during the current transaction.
IsNewContract(addr common.Address) bool
// Empty returns whether the given account is empty. Empty
// is defined according to EIP161 (balance = nonce = code = 0).
Empty(common.Address) bool

26
core/vm/operations_acl.go
View file

@ -28,6 +28,9 @@ import (
func makeGasSStoreFunc(clearingRefund uint64) gasFunc {
return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
if evm.readOnly {
return 0, ErrWriteProtection
}
// If we fail the minimum gas availability invariant, fail (0)
if contract.Gas <= params.SstoreSentryGasEIP2200 {
return 0, errors.New("not enough gas for reentrancy sentry")
@ -226,10 +229,19 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc {
gas uint64
address = common.Address(stack.peek().Bytes20())
)
if evm.readOnly {
return 0, ErrWriteProtection
}
if !evm.StateDB.AddressInAccessList(address) {
// If the caller cannot afford the cost, this change will be rolled back
evm.StateDB.AddAddressToAccessList(address)
gas = params.ColdAccountAccessCostEIP2929
// Terminate the gas measurement if the leftover gas is not sufficient,
// it can effectively prevent accessing the states in the following steps
if contract.Gas < gas {
return 0, ErrOutOfGas
}
}
// if empty and transfers value
if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 {
@ -244,12 +256,24 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc {
}
var (
gasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall)
innerGasCallEIP7702 = makeCallVariantGasCallEIP7702(gasCall)
gasDelegateCallEIP7702 = makeCallVariantGasCallEIP7702(gasDelegateCall)
gasStaticCallEIP7702 = makeCallVariantGasCallEIP7702(gasStaticCall)
gasCallCodeEIP7702 = makeCallVariantGasCallEIP7702(gasCallCode)
)
func gasCallEIP7702(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
// Return early if this call attempts to transfer value in a static context.
// Although it's checked in `gasCall`, EIP-7702 loads the target's code before
// to determine if it is resolving a delegation. This could incorrectly record
// the target in the block access list (BAL) if the call later fails.
transfersValue := !stack.Back(2).IsZero()
if evm.readOnly && transfersValue {
return 0, ErrWriteProtection
}
return innerGasCallEIP7702(evm, contract, stack, mem, memorySize)
}
func makeCallVariantGasCallEIP7702(oldCalculator gasFunc) gasFunc {
return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
var (

2
crypto/ecies/ecies.go
View file

@ -290,7 +290,7 @@ func (prv *PrivateKey) Decrypt(c, s1, s2 []byte) (m []byte, err error) {
switch c[0] {
case 2, 3, 4:
rLen = (prv.PublicKey.Curve.Params().BitSize + 7) / 4
if len(c) < (rLen + hLen + 1) {
if len(c) < (rLen + hLen + params.BlockSize) {
return nil, ErrInvalidMessage
}
default:

5
eth/api_backend.go
View file

@ -315,6 +315,11 @@ func (b *EthAPIBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) e
return b.eth.BlockChain().SubscribeChainHeadEvent(ch)
}
// SubscribeNewPayloadEvent registers a subscription for NewPayloadEvent.
func (b *EthAPIBackend) SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription {
return b.eth.BlockChain().SubscribeNewPayloadEvent(ch)
}
func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
return b.eth.BlockChain().SubscribeLogsEvent(ch)
}

1
eth/backend.go
View file

@ -230,6 +230,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateHistory: config.StateHistory,
TrienodeHistory: config.TrienodeHistory,
StateScheme: scheme,
ChainHistoryMode: config.HistoryMode,
TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)),

10
eth/catalyst/api.go
View file

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/beacon/engine"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth"
@ -787,7 +788,9 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
return engine.PayloadStatusV1{Status: engine.ACCEPTED}, nil
}
log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number())
start := time.Now()
proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness)
processingTime := time.Since(start)
if err != nil {
log.Warn("NewPayload: inserting block failed", "error", err)
@ -800,6 +803,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
}
hash := block.Hash()
// Emit NewPayloadEvent for ethstats reporting
api.eth.BlockChain().SendNewPayloadEvent(core.NewPayloadEvent{
Hash: hash,
Number: block.NumberU64(),
ProcessingTime: processingTime,
})
// If witness collection was requested, inject that into the result too
var ow *hexutil.Bytes
if proofs != nil {

6
eth/ethconfig/config.go
View file

@ -56,6 +56,7 @@ var Defaults = Config{
TransactionHistory: 2350000,
LogHistory: 2350000,
StateHistory: params.FullImmutabilityThreshold,
TrienodeHistory: -1,
DatabaseCache: 512,
TrieCleanCache: 154,
TrieDirtyCache: 256,
@ -73,6 +74,7 @@ var Defaults = Config{
TxSyncDefaultTimeout: 20 * time.Second,
TxSyncMaxTimeout: 1 * time.Minute,
SlowBlockThreshold: time.Second * 2,
RangeLimit: 0,
}
//go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go
@ -108,6 +110,7 @@ type Config struct {
LogNoHistory bool `toml:",omitempty"` // No log search index is maintained.
LogExportCheckpoints string // export log index checkpoints to file
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
TrienodeHistory int64 `toml:",omitempty"` // Number of blocks from the chain head for which trienode histories are retained
// State scheme represents the scheme used to store ethereum states and trie
// nodes on top. It can be 'hash', 'path', or none which means use the scheme
@ -194,6 +197,9 @@ type Config struct {
// EIP-7966: eth_sendRawTransactionSync timeouts
TxSyncDefaultTimeout time.Duration `toml:",omitempty"`
TxSyncMaxTimeout time.Duration `toml:",omitempty"`
// RangeLimit restricts the maximum range (end - start) for range queries.
RangeLimit uint64 `toml:",omitempty"`
}
// CreateConsensusEngine creates a consensus engine for the given chain config.

12
eth/ethconfig/gen_config.go
View file

@ -31,6 +31,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
LogNoHistory bool `toml:",omitempty"`
LogExportCheckpoints string
StateHistory uint64 `toml:",omitempty"`
TrienodeHistory int64 `toml:",omitempty"`
StateScheme string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
SlowBlockThreshold time.Duration `toml:",omitempty"`
@ -65,6 +66,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
OverrideVerkle *uint64 `toml:",omitempty"`
TxSyncDefaultTimeout time.Duration `toml:",omitempty"`
TxSyncMaxTimeout time.Duration `toml:",omitempty"`
RangeLimit uint64 `toml:",omitempty"`
}
var enc Config
enc.Genesis = c.Genesis
@ -81,6 +83,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.LogNoHistory = c.LogNoHistory
enc.LogExportCheckpoints = c.LogExportCheckpoints
enc.StateHistory = c.StateHistory
enc.TrienodeHistory = c.TrienodeHistory
enc.StateScheme = c.StateScheme
enc.RequiredBlocks = c.RequiredBlocks
enc.SlowBlockThreshold = c.SlowBlockThreshold
@ -115,6 +118,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.OverrideVerkle = c.OverrideVerkle
enc.TxSyncDefaultTimeout = c.TxSyncDefaultTimeout
enc.TxSyncMaxTimeout = c.TxSyncMaxTimeout
enc.RangeLimit = c.RangeLimit
return &enc, nil
}
@ -135,6 +139,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
LogNoHistory *bool `toml:",omitempty"`
LogExportCheckpoints *string
StateHistory *uint64 `toml:",omitempty"`
TrienodeHistory *int64 `toml:",omitempty"`
StateScheme *string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
SlowBlockThreshold *time.Duration `toml:",omitempty"`
@ -169,6 +174,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
OverrideVerkle *uint64 `toml:",omitempty"`
TxSyncDefaultTimeout *time.Duration `toml:",omitempty"`
TxSyncMaxTimeout *time.Duration `toml:",omitempty"`
RangeLimit *uint64 `toml:",omitempty"`
}
var dec Config
if err := unmarshal(&dec); err != nil {
@ -216,6 +222,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.StateHistory != nil {
c.StateHistory = *dec.StateHistory
}
if dec.TrienodeHistory != nil {
c.TrienodeHistory = *dec.TrienodeHistory
}
if dec.StateScheme != nil {
c.StateScheme = *dec.StateScheme
}
@ -318,5 +327,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.TxSyncMaxTimeout != nil {
c.TxSyncMaxTimeout = *dec.TxSyncMaxTimeout
}
if dec.RangeLimit != nil {
c.RangeLimit = *dec.RangeLimit
}
return nil
}

32
eth/fetcher/tx_fetcher.go
View file

@ -114,10 +114,11 @@ type txRequest struct {
// txDelivery is the notification that a batch of transactions have been added
// to the pool and should be untracked.
type txDelivery struct {
origin string // Identifier of the peer originating the notification
hashes []common.Hash // Batch of transaction hashes having been delivered
metas []txMetadata // Batch of metadata associated with the delivered hashes
direct bool // Whether this is a direct reply or a broadcast
origin string // Identifier of the peer originating the notification
hashes []common.Hash // Batch of transaction hashes having been delivered
metas []txMetadata // Batch of metadata associated with the delivered hashes
direct bool // Whether this is a direct reply or a broadcast
violation error // Whether we encountered a protocol violation
}
// txDrop is the notification that a peer has disconnected.
@ -292,6 +293,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
knownMeter = txReplyKnownMeter
underpricedMeter = txReplyUnderpricedMeter
otherRejectMeter = txReplyOtherRejectMeter
violation error
)
if !direct {
inMeter = txBroadcastInMeter
@ -338,6 +340,12 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow):
underpriced++
case errors.Is(err, txpool.ErrKZGVerificationError):
// KZG verification failed, terminate transaction processing immediately.
// Since KZG verification is computationally expensive, this acts as a
// defensive measure against potential DoS attacks.
violation = err
default:
otherreject++
}
@ -346,6 +354,11 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
kind: batch[j].Type(),
size: uint32(batch[j].Size()),
})
// Terminate the transaction processing if violation is encountered. All
// the remaining transactions in response will be silently discarded.
if violation != nil {
break
}
}
knownMeter.Mark(duplicate)
underpricedMeter.Mark(underpriced)
@ -356,9 +369,13 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject)
time.Sleep(200 * time.Millisecond)
}
// If we encountered a protocol violation, disconnect this peer.
if violation != nil {
break
}
}
select {
case f.cleanup <- &txDelivery{origin: peer, hashes: added, metas: metas, direct: direct}:
case f.cleanup <- &txDelivery{origin: peer, hashes: added, metas: metas, direct: direct, violation: violation}:
return nil
case <-f.quit:
return errTerminated
@ -753,6 +770,11 @@ func (f *TxFetcher) loop() {
// Something was delivered, try to reschedule requests
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
}
// If we encountered a protocol violation, disconnect the peer
if delivery.violation != nil {
log.Warn("Disconnect peer for protocol violation", "peer", delivery.origin, "error", delivery.violation)
f.dropPeer(delivery.origin)
}
case drop := <-f.drop:
// A peer was dropped, remove all traces of it

463
eth/fetcher/tx_fetcher_test.go
View file

@ -17,6 +17,7 @@
package fetcher
import (
"crypto/sha256"
"errors"
"math/big"
"math/rand"
@ -28,7 +29,10 @@ import (
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/params"
"github.com/holiman/uint256"
)
var (
@ -83,6 +87,19 @@ type txFetcherTest struct {
steps []interface{}
}
// newTestTxFetcher creates a tx fetcher with noop callbacks, simulated clock,
// and deterministic randomness.
func newTestTxFetcher() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
}
// Tests that transaction announcements with associated metadata are added to a
// waitlist, and none of them are scheduled for retrieval until the wait expires.
//
@ -91,14 +108,7 @@ type txFetcherTest struct {
// with all the useless extra fields.
func TestTransactionFetcherWaiting(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Initial announcement to get something into the waitlist
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}},
@ -293,14 +303,7 @@ func TestTransactionFetcherWaiting(t *testing.T) {
// already scheduled.
func TestTransactionFetcherSkipWaiting(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{
@ -383,14 +386,7 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) {
// and subsequent announces block or get allotted to someone else.
func TestTransactionFetcherSingletonRequesting(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}},
@ -489,15 +485,12 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) {
proceed := make(chan struct{})
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(origin string, hashes []common.Hash) error {
<-proceed
return errors.New("peer disconnected")
},
nil,
)
f := newTestTxFetcher()
f.fetchTxs = func(origin string, hashes []common.Hash) error {
<-proceed
return errors.New("peer disconnected")
}
return f
},
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
@ -572,16 +565,7 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) {
// are cleaned up.
func TestTransactionFetcherCleanup(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
@ -616,16 +600,7 @@ func TestTransactionFetcherCleanup(t *testing.T) {
// this was a bug)).
func TestTransactionFetcherCleanupEmpty(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
@ -659,16 +634,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) {
// different peer, or self if they are after the cutoff point.
func TestTransactionFetcherMissingRescheduling(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{peer: "A",
@ -720,16 +686,7 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) {
// delivered, the peer gets properly cleaned out from the internal state.
func TestTransactionFetcherMissingCleanup(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{peer: "A",
@ -769,16 +726,7 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) {
// Tests that transaction broadcasts properly clean up announcements.
func TestTransactionFetcherBroadcasts(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Set up three transactions to be in different stats, waiting, queued and fetching
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
@ -825,14 +773,7 @@ func TestTransactionFetcherBroadcasts(t *testing.T) {
// Tests that the waiting list timers properly reset and reschedule.
func TestTransactionFetcherWaitTimerResets(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}},
isWaiting(map[string][]announce{
@ -895,16 +836,7 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) {
// out and be re-scheduled for someone else.
func TestTransactionFetcherTimeoutRescheduling(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Push an initial announcement through to the scheduled stage
doTxNotify{
@ -973,14 +905,7 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) {
// Tests that the fetching timeout timers properly reset and reschedule.
func TestTransactionFetcherTimeoutTimerResets(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}},
doWait{time: txArriveTimeout, step: true},
@ -1051,14 +976,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) {
})
}
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Announce all the transactions, wait a bit and ensure only a small
// percentage gets requested
@ -1081,14 +999,7 @@ func TestTransactionFetcherRateLimiting(t *testing.T) {
// be requested at a time, to keep the responses below a reasonable level.
func TestTransactionFetcherBandwidthLimiting(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Announce mid size transactions from A to verify that multiple
// ones can be piled into a single request.
@ -1198,14 +1109,7 @@ func TestTransactionFetcherDoSProtection(t *testing.T) {
})
}
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Announce half of the transaction and wait for them to be scheduled
doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2], types: typesA[:maxTxAnnounces/2], sizes: sizesA[:maxTxAnnounces/2]},
@ -1266,24 +1170,21 @@ func TestTransactionFetcherDoSProtection(t *testing.T) {
func TestTransactionFetcherUnderpricedDedup(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
if i%3 == 0 {
errs[i] = txpool.ErrUnderpriced
} else if i%3 == 1 {
errs[i] = txpool.ErrReplaceUnderpriced
} else {
errs[i] = txpool.ErrTxGasPriceTooLow
}
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
if i%3 == 0 {
errs[i] = txpool.ErrUnderpriced
} else if i%3 == 1 {
errs[i] = txpool.ErrReplaceUnderpriced
} else {
errs[i] = txpool.ErrTxGasPriceTooLow
}
return errs
},
func(string, []common.Hash) error { return nil },
nil,
)
}
return errs
}
return f
},
steps: []interface{}{
// Deliver a transaction through the fetcher, but reject as underpriced
@ -1367,18 +1268,15 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) {
}
testTransactionFetcher(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
errs[i] = txpool.ErrUnderpriced
}
return errs
},
func(string, []common.Hash) error { return nil },
nil,
)
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
errs[i] = txpool.ErrUnderpriced
}
return errs
}
return f
},
steps: append(steps, []interface{}{
// The preparation of the test has already been done in `steps`, add the last check
@ -1398,16 +1296,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) {
// Tests that unexpected deliveries don't corrupt the internal state.
func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Deliver something out of the blue
isWaiting(nil),
@ -1457,16 +1346,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) {
// live or dangling stages.
func TestTransactionFetcherDrop(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Set up a few hashes into various stages
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}},
@ -1531,16 +1411,7 @@ func TestTransactionFetcherDrop(t *testing.T) {
// available peer.
func TestTransactionFetcherDropRescheduling(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Set up a few hashes into various stages
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}},
@ -1578,14 +1449,9 @@ func TestInvalidAnnounceMetadata(t *testing.T) {
drop := make(chan string, 2)
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
func(peer string) { drop <- peer },
)
f := newTestTxFetcher()
f.dropPeer = func(peer string) { drop <- peer }
return f
},
steps: []interface{}{
// Initial announcement to get something into the waitlist
@ -1660,16 +1526,7 @@ func TestInvalidAnnounceMetadata(t *testing.T) {
// announced one.
func TestTransactionFetcherFuzzCrash01(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Get a transaction into fetching mode and make it dangling with a broadcast
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
@ -1688,16 +1545,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) {
// concurrently announced one.
func TestTransactionFetcherFuzzCrash02(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Get a transaction into fetching mode and make it dangling with a broadcast
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
@ -1718,16 +1566,7 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) {
// with a concurrent notify.
func TestTransactionFetcherFuzzCrash03(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Get a transaction into fetching mode and make it dangling with a broadcast
doTxNotify{
@ -1758,17 +1597,12 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error {
<-proceed
return errors.New("peer disconnected")
},
nil,
)
f := newTestTxFetcher()
f.fetchTxs = func(string, []common.Hash) error {
<-proceed
return errors.New("peer disconnected")
}
return f
},
steps: []interface{}{
// Get a transaction into fetching mode and make it dangling with a broadcast
@ -1792,14 +1626,7 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) {
// once they are announced in the network.
func TestBlobTransactionAnnounce(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
nil,
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
// Initial announcement to get something into the waitlist
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}},
@ -1860,16 +1687,7 @@ func TestBlobTransactionAnnounce(t *testing.T) {
func TestTransactionFetcherDropAlternates(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
},
init: newTestTxFetcher,
steps: []interface{}{
doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}},
doWait{time: txArriveTimeout, step: true},
@ -1911,20 +1729,15 @@ func TestTransactionFetcherDropAlternates(t *testing.T) {
func TestTransactionFetcherWrongMetadata(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
return NewTxFetcher(
func(_ common.Hash, kind byte) error {
switch kind {
case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType:
return nil
}
return types.ErrTxTypeNotSupported
},
func(txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil,
)
f := newTestTxFetcher()
f.validateMeta = func(name common.Hash, kind byte) error {
switch kind {
case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType:
return nil
}
return types.ErrTxTypeNotSupported
}
return f
},
steps: []interface{}{
doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{0xff, types.LegacyTxType}, sizes: []uint32{111, 222}},
@ -1937,6 +1750,110 @@ func TestTransactionFetcherWrongMetadata(t *testing.T) {
})
}
func makeInvalidBlobTx() *types.Transaction {
key, _ := crypto.GenerateKey()
blob := &kzg4844.Blob{byte(0xa)}
commitment, _ := kzg4844.BlobToCommitment(blob)
blobHash := kzg4844.CalcBlobHashV1(sha256.New(), &commitment)
cellProof, _ := kzg4844.ComputeCellProofs(blob)
// Mutate the cell proof
cellProof[0][0] = 0x0
blobtx := &types.BlobTx{
ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID),
Nonce: 0,
GasTipCap: uint256.NewInt(100),
GasFeeCap: uint256.NewInt(200),
Gas: 21000,
BlobFeeCap: uint256.NewInt(200),
BlobHashes: []common.Hash{blobHash},
Value: uint256.NewInt(100),
Sidecar: types.NewBlobTxSidecar(types.BlobSidecarVersion1, []kzg4844.Blob{*blob}, []kzg4844.Commitment{commitment}, cellProof),
}
return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx)
}
// This test ensures that the peer will be disconnected for protocol violation
// and all its internal traces should be removed properly.
func TestTransactionProtocolViolation(t *testing.T) {
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
var (
badTx = makeInvalidBlobTx()
drop = make(chan struct{}, 1)
)
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
var errs []error
for range txs {
errs = append(errs, txpool.ErrKZGVerificationError)
}
return errs
}
f.dropPeer = func(string) { drop <- struct{}{} }
return f
},
steps: []interface{}{
// Initial announcement to get something into the waitlist
doTxNotify{
peer: "A",
hashes: []common.Hash{testTxs[0].Hash(), badTx.Hash(), testTxs[1].Hash()},
types: []byte{types.LegacyTxType, types.BlobTxType, types.LegacyTxType},
sizes: []uint32{uint32(testTxs[0].Size()), uint32(badTx.Size()), uint32(testTxs[1].Size())},
},
isWaiting(map[string][]announce{
"A": {
{testTxs[0].Hash(), types.LegacyTxType, uint32(testTxs[0].Size())},
{badTx.Hash(), types.BlobTxType, uint32(badTx.Size())},
{testTxs[1].Hash(), types.LegacyTxType, uint32(testTxs[1].Size())},
},
}),
doWait{time: 0, step: true}, // zero time, but the blob fetching should be scheduled
isWaiting(map[string][]announce{
"A": {
{testTxs[0].Hash(), types.LegacyTxType, uint32(testTxs[0].Size())},
{testTxs[1].Hash(), types.LegacyTxType, uint32(testTxs[1].Size())},
},
}),
isScheduled{
tracking: map[string][]announce{
"A": {
{badTx.Hash(), types.BlobTxType, uint32(badTx.Size())},
},
},
fetching: map[string][]common.Hash{
"A": {badTx.Hash()},
},
},
doTxEnqueue{
peer: "A",
txs: []*types.Transaction{badTx},
direct: true,
},
// Some internal traces are left and will be cleaned by a following drop
// operation.
isWaiting(map[string][]announce{
"A": {
{testTxs[0].Hash(), types.LegacyTxType, uint32(testTxs[0].Size())},
{testTxs[1].Hash(), types.LegacyTxType, uint32(testTxs[1].Size())},
},
}),
isScheduled{},
doFunc(func() { <-drop }),
// Simulate the drop operation emitted by the server
doDrop("A"),
isWaiting(nil),
isScheduled{nil, nil, nil},
},
})
}
func testTransactionFetcherParallel(t *testing.T, tt txFetcherTest) {
t.Parallel()
testTransactionFetcher(t, tt)

6
eth/filters/api.go
View file

@ -89,6 +89,7 @@ type FilterAPI struct {
filters map[rpc.ID]*filter
timeout time.Duration
logQueryLimit int
rangeLimit uint64
}
// NewFilterAPI returns a new FilterAPI instance.
@ -99,6 +100,7 @@ func NewFilterAPI(system *FilterSystem) *FilterAPI {
filters: make(map[rpc.ID]*filter),
timeout: system.cfg.Timeout,
logQueryLimit: system.cfg.LogQueryLimit,
rangeLimit: system.cfg.RangeLimit,
}
go api.timeoutLoop(system.cfg.Timeout)
@ -475,7 +477,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type
return nil, &history.PrunedHistoryError{}
}
// Construct the range filter
filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics)
filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics, api.rangeLimit)
}
// Run the filter and return all the logs
@ -527,7 +529,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo
end = f.crit.ToBlock.Int64()
}
// Construct the range filter
filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics)
filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics, api.rangeLimit)
}
// Run the filter and return all the logs
logs, err := filter.Logs(ctx)

10
eth/filters/filter.go
View file

@ -19,6 +19,7 @@ package filters
import (
"context"
"errors"
"fmt"
"math"
"math/big"
"slices"
@ -44,15 +45,17 @@ type Filter struct {
begin, end int64 // Range interval if filtering multiple blocks
rangeLogsTestHook chan rangeLogsTestEvent
rangeLimit uint64
}
// NewRangeFilter creates a new filter which uses a bloom filter on blocks to
// figure out whether a particular block is interesting or not.
func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter {
func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash, rangeLimit uint64) *Filter {
// Create a generic filter and convert it into a range filter
filter := newFilter(sys, addresses, topics)
filter.begin = begin
filter.end = end
filter.rangeLimit = rangeLimit
return filter
}
@ -151,6 +154,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
if begin > end {
return nil, errInvalidBlockRange
}
if f.rangeLimit != 0 && (end-begin) > f.rangeLimit {
return nil, fmt.Errorf("exceed maximum block range: %d", f.rangeLimit)
}
return f.rangeLogs(ctx, begin, end)
}
@ -499,7 +505,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ
// filterLogs creates a slice of logs matching the given criteria.
func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*types.Log {
var check = func(log *types.Log) bool {
check := func(log *types.Log) bool {
if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber {
return false
}

1
eth/filters/filter_system.go
View file

@ -44,6 +44,7 @@ type Config struct {
LogCacheSize int // maximum number of cached blocks (default: 32)
Timeout time.Duration // how long filters stay active (default: 5min)
LogQueryLimit int // maximum number of addresses allowed in filter criteria (default: 1000)
RangeLimit uint64 // maximum block range allowed in filter criteria (default: 0)
}
func (cfg Config) withDefaults() Config {

2
eth/filters/filter_system_test.go
View file

@ -546,7 +546,7 @@ func TestExceedLogQueryLimit(t *testing.T) {
}
)
_, err := gspec.Commit(db, triedb.NewDatabase(db, nil))
_, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil)
if err != nil {
t.Fatal(err)
}

84
eth/filters/filter_test.go
View file

@ -109,7 +109,7 @@ func benchmarkFilters(b *testing.B, history uint64, noHistory bool) {
backend.startFilterMaps(history, noHistory, filtermaps.DefaultParams)
defer backend.stopFilterMaps()
filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil)
filter := sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{addr1, addr2, addr3, addr4}, nil, 0)
for b.Loop() {
filter.begin = 0
logs, _ := filter.Logs(context.Background())
@ -205,7 +205,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) {
// Hack: GenerateChainWithGenesis creates a new db.
// Commit the genesis manually and use GenerateChain.
_, err = gspec.Commit(db, triedb.NewDatabase(db, nil))
_, err = gspec.Commit(db, triedb.NewDatabase(db, nil), nil)
if err != nil {
t.Fatal(err)
}
@ -317,75 +317,75 @@ func testFilters(t *testing.T, history uint64, noHistory bool) {
want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}),
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{contract}, [][]common.Hash{{hash1, hash2, hash3, hash4}}, 0),
want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}),
f: sys.NewRangeFilter(900, 999, []common.Address{contract}, [][]common.Hash{{hash3}}, 0),
},
{
f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}),
f: sys.NewRangeFilter(990, int64(rpc.LatestBlockNumber), []common.Address{contract2}, [][]common.Hash{{hash3}}, 0),
want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}),
f: sys.NewRangeFilter(1, 10, []common.Address{contract}, [][]common.Hash{{hash2}, {hash1}}, 0),
want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}),
f: sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}, 0),
want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xa8028c655b6423204c8edfbc339f57b042d6bec2b6a61145d76b7c08b4cccd42","transactionIndex":"0x0","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x0","removed":false},{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x2","transactionHash":"0xdba3e2ea9a7d690b722d70ee605fd67ba4c00d1d3aecd5cf187a7b92ad8eb3df","transactionIndex":"0x1","blockHash":"0x24417bb49ce44cfad65da68f33b510bf2a129c0d89ccf06acb6958b8585ccf34","blockTimestamp":"0x14","logIndex":"0x1","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696332","0x0000000000000000000000000000000000000000000000000000746f70696331"],"data":"0x","blockNumber":"0x3","transactionHash":"0xdefe471992a07a02acdfbe33edaae22fbb86d7d3cec3f1b8e4e77702fb3acc1d","transactionIndex":"0x0","blockHash":"0x7a7556792ca7d37882882e2b001fe14833eaf81c2c7f865c9c771ec37a024f6b","blockTimestamp":"0x1e","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}),
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}}, 0),
},
{
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil),
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), []common.Address{common.BytesToAddress([]byte("failmenow"))}, nil, 0),
},
{
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}),
f: sys.NewRangeFilter(0, int64(rpc.LatestBlockNumber), nil, [][]common.Hash{{common.BytesToHash([]byte("fail"))}, {hash1}}, 0),
},
{
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0),
want: `[{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0),
want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false},{"address":"0xfe00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696334"],"data":"0x","blockNumber":"0x3e8","transactionHash":"0x9a87842100a638dfa5da8842b4beda691d2fd77b0c84b57f24ecfa9fb208f747","transactionIndex":"0x0","blockHash":"0xb360bad5265261c075ece02d3bf0e39498a6a76310482cdfd90588748e6c5ee0","blockTimestamp":"0x2710","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.FinalizedBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0),
want: `[{"address":"0xff00000000000000000000000000000000000000","topics":["0x0000000000000000000000000000000000000000000000000000746f70696333"],"data":"0x","blockNumber":"0x3e7","transactionHash":"0x53e3675800c6908424b61b35a44e51ca4c73ca603e58a65b32c67968b4f42200","transactionIndex":"0x0","blockHash":"0x2e4620a2b426b0612ec6cad9603f466723edaed87f98c9137405dd4f7a2409ff","blockTimestamp":"0x2706","logIndex":"0x0","removed":false}]`,
},
{
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.FinalizedBlockNumber), nil, nil, 0),
err: "invalid block range params",
},
{
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.EarliestBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.EarliestBlockNumber), nil, nil, 0),
err: "invalid block range params",
},
{
f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0),
err: "safe header not found",
},
{
f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.SafeBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0),
err: "safe header not found",
},
{
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.SafeBlockNumber), nil, nil, 0),
err: "safe header not found",
},
{
f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0),
err: errPendingLogsUnsupported.Error(),
},
{
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.LatestBlockNumber), int64(rpc.PendingBlockNumber), nil, nil, 0),
err: errPendingLogsUnsupported.Error(),
},
{
f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil),
f: sys.NewRangeFilter(int64(rpc.PendingBlockNumber), int64(rpc.LatestBlockNumber), nil, nil, 0),
err: errPendingLogsUnsupported.Error(),
},
} {
@ -408,7 +408,7 @@ func testFilters(t *testing.T, history uint64, noHistory bool) {
}
t.Run("timeout", func(t *testing.T) {
f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil)
f := sys.NewRangeFilter(0, rpc.LatestBlockNumber.Int64(), nil, nil, 0)
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour))
defer cancel()
_, err := f.Logs(ctx)
@ -431,7 +431,7 @@ func TestRangeLogs(t *testing.T) {
BaseFee: big.NewInt(params.InitialBaseFee),
}
)
_, err := gspec.Commit(db, triedb.NewDatabase(db, nil))
_, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil)
if err != nil {
t.Fatal(err)
}
@ -469,7 +469,7 @@ func TestRangeLogs(t *testing.T) {
newFilter := func(begin, end int64) {
testCase++
event = 0
filter = sys.NewRangeFilter(begin, end, addresses, nil)
filter = sys.NewRangeFilter(begin, end, addresses, nil, 0)
filter.rangeLogsTestHook = make(chan rangeLogsTestEvent)
go func(filter *Filter) {
filter.Logs(context.Background())
@ -606,3 +606,39 @@ func TestRangeLogs(t *testing.T) {
expEvent(rangeLogsTestReorg, 400, 901)
expEvent(rangeLogsTestDone, 0, 0)
}
func TestRangeLimit(t *testing.T) {
db := rawdb.NewMemoryDatabase()
backend, sys := newTestFilterSystem(db, Config{})
defer db.Close()
gspec := &core.Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{},
BaseFee: big.NewInt(params.InitialBaseFee),
}
_, err := gspec.Commit(db, triedb.NewDatabase(db, nil), nil)
if err != nil {
t.Fatal(err)
}
chain, _ := core.GenerateChain(gspec.Config, gspec.ToBlock(), ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {})
options := core.DefaultConfig().WithStateScheme(rawdb.HashScheme)
options.TxLookupLimit = 0
bc, err := core.NewBlockChain(db, gspec, ethash.NewFaker(), options)
if err != nil {
t.Fatal(err)
}
_, err = bc.InsertChain(chain)
if err != nil {
t.Fatal(err)
}
backend.startFilterMaps(0, false, filtermaps.DefaultParams)
defer backend.stopFilterMaps()
// Set rangeLimit to 5, but request a range of 9 (end - begin = 9, from 0 to 9)
filter := sys.NewRangeFilter(0, 9, nil, nil, 5)
_, err = filter.Logs(context.Background())
if err == nil || !strings.Contains(err.Error(), "exceed maximum block range") {
t.Fatalf("expected range limit error, got %v", err)
}
}

2
eth/handler.go
View file

@ -509,7 +509,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) {
annCount += len(hashes)
peer.AsyncSendPooledTransactionHashes(hashes)
}
log.Debug("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs,
log.Trace("Distributed transactions", "plaintxs", len(txs)-blobTxs-largeTxs, "blobtxs", blobTxs, "largetxs", largeTxs,
"bcastpeers", len(txset), "bcastcount", directCount, "annpeers", len(annos), "anncount", annCount)
}

2
eth/protocols/eth/peer.go
View file

@ -341,7 +341,7 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ
// RequestTxs fetches a batch of transactions from a remote node.
func (p *Peer) RequestTxs(hashes []common.Hash) error {
p.Log().Debug("Fetching batch of transactions", "count", len(hashes))
p.Log().Trace("Fetching batch of transactions", "count", len(hashes))
id := rand.Uint64()
requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id)

653
eth/tracers/internal/tracetest/selfdestruct_state_test.go Normal file
View file

@ -0,0 +1,653 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package tracetest
import (
"math/big"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/tracing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
)
// accountState represents the expected final state of an account
type accountState struct {
Balance *big.Int
Nonce uint64
Code []byte
Exists bool
}
// selfdestructStateTracer tracks state changes during selfdestruct operations
type selfdestructStateTracer struct {
env *tracing.VMContext
accounts map[common.Address]*accountState
}
func newSelfdestructStateTracer() *selfdestructStateTracer {
return &selfdestructStateTracer{
accounts: make(map[common.Address]*accountState),
}
}
func (t *selfdestructStateTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) {
t.env = env
}
func (t *selfdestructStateTracer) OnTxEnd(receipt *types.Receipt, err error) {
// Nothing to do
}
func (t *selfdestructStateTracer) getOrCreateAccount(addr common.Address) *accountState {
if acc, ok := t.accounts[addr]; ok {
return acc
}
// Initialize with current state from statedb
acc := &accountState{
Balance: t.env.StateDB.GetBalance(addr).ToBig(),
Nonce: t.env.StateDB.GetNonce(addr),
Code: t.env.StateDB.GetCode(addr),
Exists: t.env.StateDB.Exist(addr),
}
t.accounts[addr] = acc
return acc
}
func (t *selfdestructStateTracer) OnBalanceChange(addr common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) {
acc := t.getOrCreateAccount(addr)
acc.Balance = new
}
func (t *selfdestructStateTracer) OnNonceChangeV2(addr common.Address, prev, new uint64, reason tracing.NonceChangeReason) {
acc := t.getOrCreateAccount(addr)
acc.Nonce = new
// If this is a selfdestruct nonce change, mark account as not existing
if reason == tracing.NonceChangeSelfdestruct {
acc.Exists = false
}
}
func (t *selfdestructStateTracer) OnCodeChangeV2(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) {
acc := t.getOrCreateAccount(addr)
acc.Code = code
// If this is a selfdestruct code change, mark account as not existing
if reason == tracing.CodeChangeSelfDestruct {
acc.Exists = false
}
}
func (t *selfdestructStateTracer) Hooks() *tracing.Hooks {
return &tracing.Hooks{
OnTxStart: t.OnTxStart,
OnTxEnd: t.OnTxEnd,
OnBalanceChange: t.OnBalanceChange,
OnNonceChangeV2: t.OnNonceChangeV2,
OnCodeChangeV2: t.OnCodeChangeV2,
}
}
func (t *selfdestructStateTracer) Accounts() map[common.Address]*accountState {
return t.accounts
}
// verifyAccountState compares actual and expected account state and reports any mismatches
func verifyAccountState(t *testing.T, addr common.Address, actual, expected *accountState) {
if actual.Balance.Cmp(expected.Balance) != 0 {
t.Errorf("address %s: balance mismatch: have %s, want %s",
addr.Hex(), actual.Balance, expected.Balance)
}
if actual.Nonce != expected.Nonce {
t.Errorf("address %s: nonce mismatch: have %d, want %d",
addr.Hex(), actual.Nonce, expected.Nonce)
}
if len(actual.Code) != len(expected.Code) {
t.Errorf("address %s: code length mismatch: have %d, want %d",
addr.Hex(), len(actual.Code), len(expected.Code))
}
if actual.Exists != expected.Exists {
t.Errorf("address %s: exists mismatch: have %v, want %v",
addr.Hex(), actual.Exists, expected.Exists)
}
}
// setupTestBlockchain creates a blockchain with the given genesis and transaction,
// returns the blockchain, the first block, and a statedb at genesis for testing
func setupTestBlockchain(t *testing.T, genesis *core.Genesis, tx *types.Transaction, useBeacon bool) (*core.BlockChain, *types.Block, *state.StateDB) {
var engine consensus.Engine
if useBeacon {
engine = beacon.New(ethash.NewFaker())
} else {
engine = ethash.NewFaker()
}
_, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, 1, func(i int, b *core.BlockGen) {
b.AddTx(tx)
})
db := rawdb.NewMemoryDatabase()
blockchain, err := core.NewBlockChain(db, genesis, engine, nil)
if err != nil {
t.Fatalf("failed to create blockchain: %v", err)
}
if _, err := blockchain.InsertChain(blocks); err != nil {
t.Fatalf("failed to insert chain: %v", err)
}
genesisBlock := blockchain.GetBlockByNumber(0)
if genesisBlock == nil {
t.Fatalf("failed to get genesis block")
}
statedb, err := blockchain.StateAt(genesisBlock.Root())
if err != nil {
t.Fatalf("failed to get state: %v", err)
}
return blockchain, blocks[0], statedb
}
func TestSelfdestructStateTracer(t *testing.T) {
t.Parallel()
const (
// Gas limit high enough for all test scenarios (factory creation + multiple calls)
testGasLimit = 500000
// Common balance amounts used across tests
testBalanceInitial = 100 // Initial balance for contracts being tested
testBalanceSent = 50 // Amount sent back in sendback tests
testBalanceFactory = 200 // Factory needs extra balance for contract creation
)
// Helper to create *big.Int for wei amounts
wei := func(amount int64) *big.Int {
return big.NewInt(amount)
}
// Test account (transaction sender)
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
caller = crypto.PubkeyToAddress(key.PublicKey)
)
// Simple selfdestruct test contracts
var (
contract = common.HexToAddress("0x00000000000000000000000000000000000000bb")
recipient = common.HexToAddress("0x00000000000000000000000000000000000000cc")
)
// Build selfdestruct code: PUSH20 <recipient> SELFDESTRUCT
selfdestructCode := []byte{byte(vm.PUSH20)}
selfdestructCode = append(selfdestructCode, recipient.Bytes()...)
selfdestructCode = append(selfdestructCode, byte(vm.SELFDESTRUCT))
// Factory test contracts (create-and-destroy pattern)
var (
factory = common.HexToAddress("0x00000000000000000000000000000000000000ff")
)
// Factory code: creates a contract with 100 wei and calls it to trigger selfdestruct back to factory
// See selfdestruct_test_contracts/factory.yul for source
// Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factory.yul --bin
// (Using paris to avoid PUSH0 opcode which is not available pre-Shanghai)
var (
factoryCode = common.Hex2Bytes("6a6133ff6000526002601ef360a81b600052600080808080600b816064f05af100")
createdContractAddr = crypto.CreateAddress(factory, 0) // Address where factory creates the contract
)
// Sendback test contracts (A→B→A pattern)
// For the refund test: Coordinator calls A, then B
// A selfdestructs to B, B sends funds back to A
var (
contractA = common.HexToAddress("0x00000000000000000000000000000000000000aa")
contractB = common.HexToAddress("0x00000000000000000000000000000000000000bb")
coordinator = common.HexToAddress("0x00000000000000000000000000000000000000cc")
)
// Contract A: if msg.value > 0, accept funds; else selfdestruct to B
// See selfdestruct_test_contracts/contractA.yul for source
// Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractA.yul --bin
contractACode := common.Hex2Bytes("60003411600a5760bbff5b00")
// Contract B: sends 50 wei back to contract A
// See selfdestruct_test_contracts/contractB.yul for source
// Runtime bytecode compiled with: solc --strict-assembly --evm-version paris contractB.yul --bin
contractBCode := common.Hex2Bytes("6000808080603260aa5af100")
// Coordinator: calls A (A selfdestructs to B), then calls B (B sends funds to A)
// See selfdestruct_test_contracts/coordinator.yul for source
// Runtime bytecode compiled with: solc --strict-assembly --evm-version paris coordinator.yul --bin
coordinatorCode := common.Hex2Bytes("60008080808060aa818080808060bb955af1505af100")
// Factory for create-and-refund test: creates A with 100 wei, calls A, calls B
// See selfdestruct_test_contracts/factoryRefund.yul for source
// Runtime bytecode compiled with: solc --strict-assembly --evm-version paris factoryRefund.yul --bin
var (
factoryRefund = common.HexToAddress("0x00000000000000000000000000000000000000dd")
factoryRefundCode = common.Hex2Bytes("60008080808060bb78600c600d600039600c6000f3fe60003411600a5760bbff5b0082528180808080601960076064f05af1505af100")
createdContractAddrA = crypto.CreateAddress(factoryRefund, 0) // Address where factory creates contract A
)
// Self-destruct-to-self test contracts
var (
contractSelfDestruct = common.HexToAddress("0x00000000000000000000000000000000000000aa")
coordinatorSendAfter = common.HexToAddress("0x00000000000000000000000000000000000000ee")
)
// Contract that selfdestructs to self
// See selfdestruct_test_contracts/contractSelfDestruct.yul
contractSelfDestructCode := common.Hex2Bytes("30ff")
// Coordinator: calls contract (triggers selfdestruct to self), stores balance, sends 50 wei, stores balance again
// See selfdestruct_test_contracts/coordinatorSendAfter.yul
coordinatorSendAfterCode := common.Hex2Bytes("60aa600080808080855af150803160005560008080806032855af1503160015500")
// Factory with balance checking: creates contract, calls it, checks balances
// See selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul
var (
factorySelfDestructBalanceCheck = common.HexToAddress("0x00000000000000000000000000000000000000fd")
factorySelfDestructBalanceCheckCode = common.Hex2Bytes("6e6002600d60003960026000f3fe30ff600052600f60116064f0600080808080855af150803160005560008080806032855af1503160015500")
createdContractAddrSelfBalanceCheck = crypto.CreateAddress(factorySelfDestructBalanceCheck, 0)
)
tests := []struct {
name string
description string
targetContract common.Address
genesis *core.Genesis
useBeacon bool
expectedResults map[common.Address]accountState
expectedStorage map[common.Address]map[uint64]*big.Int
}{
{
name: "pre_6780_existing",
description: "Pre-EIP-6780: Existing contract selfdestructs to recipient. Contract should be destroyed and balance transferred.",
targetContract: contract,
genesis: &core.Genesis{
Config: params.AllEthashProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contract: {
Balance: wei(testBalanceInitial),
Code: selfdestructCode,
},
},
},
useBeacon: false,
expectedResults: map[common.Address]accountState{
contract: {
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
recipient: {
Balance: wei(testBalanceInitial), // Received contract's balance
Nonce: 0,
Code: []byte{},
Exists: true,
},
},
},
{
name: "post_6780_existing",
description: "Post-EIP-6780: Existing contract selfdestructs to recipient. Balance transferred but contract NOT destroyed (code/storage remain).",
targetContract: contract,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contract: {
Balance: wei(testBalanceInitial),
Code: selfdestructCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
contract: {
Balance: wei(0),
Nonce: 0,
Code: selfdestructCode,
Exists: true,
},
recipient: {
Balance: wei(testBalanceInitial),
Nonce: 0,
Code: []byte{},
Exists: true,
},
},
},
{
name: "pre_6780_create_destroy",
description: "Pre-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed, factory gets refund.",
targetContract: factory,
genesis: &core.Genesis{
Config: params.AllEthashProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
factory: {
Balance: wei(testBalanceFactory),
Code: factoryCode,
},
},
},
useBeacon: false,
expectedResults: map[common.Address]accountState{
factory: {
Balance: wei(testBalanceFactory),
Nonce: 1,
Code: factoryCode,
Exists: true,
},
createdContractAddr: {
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
},
},
{
name: "post_6780_create_destroy",
description: "Post-EIP-6780: Factory creates contract with 100 wei, contract selfdestructs back to factory. Contract destroyed (EIP-6780 exception for same-tx creation).",
targetContract: factory,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
factory: {
Balance: wei(testBalanceFactory),
Code: factoryCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
factory: {
Balance: wei(testBalanceFactory),
Nonce: 1,
Code: factoryCode,
Exists: true,
},
createdContractAddr: {
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
},
},
{
name: "pre_6780_sendback",
description: "Pre-EIP-6780: Contract A selfdestructs sending funds to B, then B sends funds back to A's address. Funds sent to destroyed address are burnt.",
targetContract: coordinator,
genesis: &core.Genesis{
Config: params.AllEthashProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contractA: {
Balance: wei(testBalanceInitial),
Code: contractACode,
},
contractB: {
Balance: wei(0),
Code: contractBCode,
},
coordinator: {
Code: coordinatorCode,
},
},
},
useBeacon: false,
expectedResults: map[common.Address]accountState{
contractA: {
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
contractB: {
// 100 received - 50 sent back
Balance: wei(testBalanceSent),
Nonce: 0,
Code: contractBCode,
Exists: true,
},
},
},
{
name: "post_6780_existing_sendback",
description: "Post-EIP-6780: Existing contract A selfdestructs to B, then B sends funds back to A. Funds are NOT burnt (A still exists post-6780).",
targetContract: coordinator,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contractA: {
Balance: wei(testBalanceInitial),
Code: contractACode,
},
contractB: {
Balance: wei(0),
Code: contractBCode,
},
coordinator: {
Code: coordinatorCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
contractA: {
Balance: wei(testBalanceSent),
Nonce: 0,
Code: contractACode,
Exists: true,
},
contractB: {
Balance: wei(testBalanceSent),
Nonce: 0,
Code: contractBCode,
Exists: true,
},
},
},
{
name: "post_6780_create_destroy_sendback",
description: "Post-EIP-6780: Factory creates A, A selfdestructs to B, B sends funds back to A. Funds are burnt (A was destroyed via EIP-6780 exception).",
targetContract: factoryRefund,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contractB: {
Balance: wei(0),
Code: contractBCode,
},
factoryRefund: {
Balance: wei(testBalanceFactory),
Code: factoryRefundCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
createdContractAddrA: {
// Funds sent back are burnt!
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
contractB: {
Balance: wei(testBalanceSent),
Nonce: 0,
Code: contractBCode,
Exists: true,
},
},
},
{
name: "post_6780_existing_to_self",
description: "Post-EIP-6780: Pre-existing contract selfdestructs to itself. Balance NOT burnt (selfdestruct-to-self is no-op for existing contracts).",
targetContract: coordinatorSendAfter,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
contractSelfDestruct: {
Balance: wei(testBalanceInitial),
Code: contractSelfDestructCode,
},
coordinatorSendAfter: {
Balance: wei(testBalanceInitial),
Code: coordinatorSendAfterCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
contractSelfDestruct: {
Balance: wei(150),
Nonce: 0,
Code: contractSelfDestructCode,
Exists: true,
},
coordinatorSendAfter: {
Balance: wei(testBalanceSent),
Nonce: 0,
Code: coordinatorSendAfterCode,
Exists: true,
},
},
expectedStorage: map[common.Address]map[uint64]*big.Int{
coordinatorSendAfter: {
0: wei(testBalanceInitial),
1: wei(150),
},
},
},
{
name: "post_6780_create_destroy_to_self",
description: "Post-EIP-6780: Factory creates contract, contract selfdestructs to itself. Balance IS burnt and contract destroyed (EIP-6780 exception for same-tx creation).",
targetContract: factorySelfDestructBalanceCheck,
genesis: &core.Genesis{
Config: params.AllDevChainProtocolChanges,
Alloc: types.GenesisAlloc{
caller: {Balance: big.NewInt(params.Ether)},
factorySelfDestructBalanceCheck: {
Balance: wei(testBalanceFactory),
Code: factorySelfDestructBalanceCheckCode,
},
},
},
useBeacon: true,
expectedResults: map[common.Address]accountState{
createdContractAddrSelfBalanceCheck: {
Balance: wei(0),
Nonce: 0,
Code: []byte{},
Exists: false,
},
factorySelfDestructBalanceCheck: {
Balance: wei(testBalanceSent),
Nonce: 1,
Code: factorySelfDestructBalanceCheckCode,
Exists: true,
},
},
expectedStorage: map[common.Address]map[uint64]*big.Int{
factorySelfDestructBalanceCheck: {
0: wei(0),
1: wei(0),
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var (
signer = types.HomesteadSigner{}
tx *types.Transaction
err error
)
tx, err = types.SignTx(types.NewTx(&types.LegacyTx{
Nonce: 0,
To: &tt.targetContract,
Value: big.NewInt(0),
Gas: testGasLimit,
GasPrice: big.NewInt(params.InitialBaseFee * 2),
Data: nil,
}), signer, key)
if err != nil {
t.Fatalf("failed to sign transaction: %v", err)
}
blockchain, block, statedb := setupTestBlockchain(t, tt.genesis, tx, tt.useBeacon)
defer blockchain.Stop()
tracer := newSelfdestructStateTracer()
hookedState := state.NewHookedState(statedb, tracer.Hooks())
msg, err := core.TransactionToMessage(tx, signer, nil)
if err != nil {
t.Fatalf("failed to prepare transaction for tracing: %v", err)
}
context := core.NewEVMBlockContext(block.Header(), blockchain, nil)
evm := vm.NewEVM(context, hookedState, tt.genesis.Config, vm.Config{Tracer: tracer.Hooks()})
usedGas := uint64(0)
_, err = core.ApplyTransactionWithEVM(msg, new(core.GasPool).AddGas(tx.Gas()), statedb, block.Number(), block.Hash(), block.Time(), tx, &usedGas, evm)
if err != nil {
t.Fatalf("failed to execute transaction: %v", err)
}
results := tracer.Accounts()
// Verify storage
for addr, expectedSlots := range tt.expectedStorage {
for slot, expectedValue := range expectedSlots {
actualValue := statedb.GetState(addr, common.BigToHash(big.NewInt(int64(slot))))
if actualValue.Big().Cmp(expectedValue) != 0 {
t.Errorf("address %s slot %d: storage mismatch: have %s, want %s",
addr.Hex(), slot, actualValue.Big(), expectedValue)
}
}
}
// Verify results
for addr, expected := range tt.expectedResults {
actual, ok := results[addr]
if !ok {
t.Errorf("address %s missing from results", addr.Hex())
continue
}
verifyAccountState(t, addr, actual, &expected)
}
})
}
}

18
eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractA.yul Normal file
View file

@ -0,0 +1,18 @@
object "ContractA" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
// If receiving funds (msg.value > 0), just accept them and return
if gt(callvalue(), 0) {
stop()
}
// Otherwise, selfdestruct to B (transfers balance immediately, then stops execution)
let contractB := 0x00000000000000000000000000000000000000bb
selfdestruct(contractB)
}
}
}

14
eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractB.yul Normal file
View file

@ -0,0 +1,14 @@
object "ContractB" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
// Send 50 wei back to contract A
let contractA := 0x00000000000000000000000000000000000000aa
let success := call(gas(), contractA, 50, 0, 0, 0, 0)
stop()
}
}
}

12
eth/tracers/internal/tracetest/selfdestruct_test_contracts/contractSelfDestruct.yul Normal file
View file

@ -0,0 +1,12 @@
object "ContractSelfDestruct" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
// Simply selfdestruct to self
selfdestruct(address())
}
}
}

20
eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinator.yul Normal file
View file

@ -0,0 +1,20 @@
object "Coordinator" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
let contractA := 0x00000000000000000000000000000000000000aa
let contractB := 0x00000000000000000000000000000000000000bb
// First, call A (A will selfdestruct to B)
pop(call(gas(), contractA, 0, 0, 0, 0, 0))
// Then, call B (B will send funds back to A)
pop(call(gas(), contractB, 0, 0, 0, 0, 0))
stop()
}
}
}

27
eth/tracers/internal/tracetest/selfdestruct_test_contracts/coordinatorSendAfter.yul Normal file
View file

@ -0,0 +1,27 @@
object "CoordinatorSendAfter" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
let contractAddr := 0x00000000000000000000000000000000000000aa
// Call contract (triggers selfdestruct to self, burning its balance)
pop(call(gas(), contractAddr, 0, 0, 0, 0, 0))
// Check contract's balance immediately after selfdestruct
// Store in slot 0 to verify it's 0 (proving immediate burn)
sstore(0, balance(contractAddr))
// Send 50 wei to the contract (after it selfdestructed)
pop(call(gas(), contractAddr, 50, 0, 0, 0, 0))
// Check balance again after sending funds
// Store in slot 1 to verify it's 50 (new funds not burnt)
sstore(1, balance(contractAddr))
stop()
}
}
}

28
eth/tracers/internal/tracetest/selfdestruct_test_contracts/factoryRefund.yul Normal file
View file

@ -0,0 +1,28 @@
object "FactoryRefund" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
let contractB := 0x00000000000000000000000000000000000000bb
// Store the deploy bytecode for contract A in memory
// Full deploy bytecode from: solc --strict-assembly --evm-version paris contractA.yul --bin
// Including the 0xfe separator: 600c600d600039600c6000f3fe60003411600a5760bbff5b00
// That's 25 bytes, padded to 32 bytes with 7 zero bytes at the front
mstore(0, 0x0000000000000000000000000000600c600d600039600c6000f3fe60003411600a5760bbff5b00)
// CREATE contract A with 100 wei, using 25 bytes starting at position 7
let contractA := create(100, 7, 25)
// Call contract A (triggers selfdestruct to B)
pop(call(gas(), contractA, 0, 0, 0, 0, 0))
// Call contract B (B sends 50 wei back to A)
pop(call(gas(), contractB, 0, 0, 0, 0, 0))
stop()
}
}
}

35
eth/tracers/internal/tracetest/selfdestruct_test_contracts/factorySelfDestructBalanceCheck.yul Normal file
View file

@ -0,0 +1,35 @@
object "FactorySelfDestructBalanceCheck" {
code {
datacopy(0, dataoffset("Runtime"), datasize("Runtime"))
return(0, datasize("Runtime"))
}
object "Runtime" {
code {
// Get the full deploy bytecode for ContractSelfDestruct
// Compiled with: solc --strict-assembly --evm-version paris contractSelfDestruct.yul --bin
// Full bytecode: 6002600d60003960026000f3fe30ff
// That's 15 bytes total, padded to 32 bytes with 17 zero bytes at front
mstore(0, 0x0000000000000000000000000000000000000000006002600d60003960026000f3fe30ff)
// CREATE contract with 100 wei, using deploy bytecode
// The bytecode is 15 bytes, starts at position 17 in the 32-byte word
let contractAddr := create(100, 17, 15)
// Call the created contract (triggers selfdestruct to self)
pop(call(gas(), contractAddr, 0, 0, 0, 0, 0))
// Check contract's balance immediately after selfdestruct
// Store in slot 0 to verify it's 0 (proving immediate burn)
sstore(0, balance(contractAddr))
// Send 50 wei to the contract (after it selfdestructed)
pop(call(gas(), contractAddr, 50, 0, 0, 0, 0))
// Check balance again after sending funds
// Store in slot 1 to verify it's 0 (funds sent to destroyed contract are burnt)
sstore(1, balance(contractAddr))
stop()
}
}
}

11
ethclient/ethclient.go
View file

@ -124,7 +124,7 @@ func (ec *Client) PeerCount(ctx context.Context) (uint64, error) {
// BlockReceipts returns the receipts of a given block number or hash.
func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.Receipt, error) {
var r []*types.Receipt
err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash.String())
err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash)
if err == nil && r == nil {
return nil, ethereum.NotFound
}
@ -497,9 +497,12 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer
}
func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
arg := map[string]interface{}{
"address": q.Addresses,
"topics": q.Topics,
arg := map[string]interface{}{}
if q.Addresses != nil {
arg["address"] = q.Addresses
}
if q.Topics != nil {
arg["topics"] = q.Topics
}
if q.BlockHash != nil {
arg["blockHash"] = *q.BlockHash

43
ethclient/ethclient_test.go
View file

@ -687,6 +687,49 @@ func testTransactionSender(t *testing.T, client *rpc.Client) {
}
}
func TestBlockReceiptsPreservesCanonicalFlag(t *testing.T) {
srv := rpc.NewServer()
service := &blockReceiptsTestService{calls: make(chan rpc.BlockNumberOrHash, 1)}
if err := srv.RegisterName("eth", service); err != nil {
t.Fatalf("failed to register service: %v", err)
}
defer srv.Stop()
client := rpc.DialInProc(srv)
defer client.Close()
ec := ethclient.NewClient(client)
defer ec.Close()
hash := common.HexToHash("0x01")
ref := rpc.BlockNumberOrHashWithHash(hash, true)
if _, err := ec.BlockReceipts(context.Background(), ref); err != nil {
t.Fatalf("BlockReceipts returned error: %v", err)
}
select {
case call := <-service.calls:
if call.BlockHash == nil || *call.BlockHash != hash {
t.Fatalf("unexpected block hash: got %v, want %v", call.BlockHash, hash)
}
if !call.RequireCanonical {
t.Fatalf("requireCanonical flag was lost: %+v", call)
}
default:
t.Fatal("service was not called")
}
}
type blockReceiptsTestService struct {
calls chan rpc.BlockNumberOrHash
}
func (s *blockReceiptsTestService) GetBlockReceipts(ctx context.Context, block rpc.BlockNumberOrHash) ([]*types.Receipt, error) {
s.calls <- block
return []*types.Receipt{}, nil
}
func newCanceledContext() context.Context {
ctx, cancel := context.WithCancel(context.Background())
cancel()

12
ethclient/types_test.go
View file

@ -41,6 +41,18 @@ func TestToFilterArg(t *testing.T) {
output interface{}
err error
}{
{
"without addresses",
ethereum.FilterQuery{
FromBlock: big.NewInt(1),
ToBlock: big.NewInt(2),
},
map[string]interface{}{
"fromBlock": "0x1",
"toBlock": "0x2",
},
nil,
},
{
"without BlockHash",
ethereum.FilterQuery{

60
ethstats/ethstats.go
View file

@ -63,6 +63,7 @@ const (
type backend interface {
SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription
SubscribeNewPayloadEvent(ch chan<- core.NewPayloadEvent) event.Subscription
CurrentHeader() *types.Header
HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
Stats() (pending int, queued int)
@ -92,8 +93,9 @@ type Service struct {
pongCh chan struct{} // Pong notifications are fed into this channel
histCh chan []uint64 // History request block numbers are fed into this channel
headSub event.Subscription
txSub event.Subscription
headSub event.Subscription
txSub event.Subscription
newPayloadSub event.Subscription
}
// connWrapper is a wrapper to prevent concurrent-write or concurrent-read on the
@ -198,7 +200,9 @@ func (s *Service) Start() error {
s.headSub = s.backend.SubscribeChainHeadEvent(chainHeadCh)
txEventCh := make(chan core.NewTxsEvent, txChanSize)
s.txSub = s.backend.SubscribeNewTxsEvent(txEventCh)
go s.loop(chainHeadCh, txEventCh)
newPayloadCh := make(chan core.NewPayloadEvent, chainHeadChanSize)
s.newPayloadSub = s.backend.SubscribeNewPayloadEvent(newPayloadCh)
go s.loop(chainHeadCh, txEventCh, newPayloadCh)
log.Info("Stats daemon started")
return nil
@ -208,18 +212,20 @@ func (s *Service) Start() error {
func (s *Service) Stop() error {
s.headSub.Unsubscribe()
s.txSub.Unsubscribe()
s.newPayloadSub.Unsubscribe()
log.Info("Stats daemon stopped")
return nil
}
// loop keeps trying to connect to the netstats server, reporting chain events
// until termination.
func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent) {
func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core.NewTxsEvent, newPayloadCh chan core.NewPayloadEvent) {
// Start a goroutine that exhausts the subscriptions to avoid events piling up
var (
quitCh = make(chan struct{})
headCh = make(chan *types.Header, 1)
txCh = make(chan struct{}, 1)
quitCh = make(chan struct{})
headCh = make(chan *types.Header, 1)
txCh = make(chan struct{}, 1)
newPayloadEvCh = make(chan core.NewPayloadEvent, 1)
)
go func() {
var lastTx mclock.AbsTime
@ -246,11 +252,20 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core
default:
}
// Notify of new payload events, but drop if too frequent
case ev := <-newPayloadCh:
select {
case newPayloadEvCh <- ev:
default:
}
// node stopped
case <-s.txSub.Err():
break HandleLoop
case <-s.headSub.Err():
break HandleLoop
case <-s.newPayloadSub.Err():
break HandleLoop
}
}
close(quitCh)
@ -336,6 +351,10 @@ func (s *Service) loop(chainHeadCh chan core.ChainHeadEvent, txEventCh chan core
if err = s.reportPending(conn); err != nil {
log.Warn("Post-block transaction stats report failed", "err", err)
}
case ev := <-newPayloadEvCh:
if err = s.reportNewPayload(conn, ev); err != nil {
log.Warn("New payload stats report failed", "err", err)
}
case <-txCh:
if err = s.reportPending(conn); err != nil {
log.Warn("Transaction stats report failed", "err", err)
@ -600,6 +619,33 @@ func (s uncleStats) MarshalJSON() ([]byte, error) {
return []byte("[]"), nil
}
// newPayloadStats is the information to report about new payload events.
type newPayloadStats struct {
Number uint64 `json:"number"`
Hash common.Hash `json:"hash"`
ProcessingTime uint64 `json:"processingTime"` // nanoseconds
}
// reportNewPayload reports a new payload event to the stats server.
func (s *Service) reportNewPayload(conn *connWrapper, ev core.NewPayloadEvent) error {
details := &newPayloadStats{
Number: ev.Number,
Hash: ev.Hash,
ProcessingTime: uint64(ev.ProcessingTime.Nanoseconds()),
}
log.Trace("Sending new payload to ethstats", "number", details.Number, "hash", details.Hash)
stats := map[string]interface{}{
"id": s.node,
"block": details,
}
report := map[string][]interface{}{
"emit": {"block_new_payload", stats},
}
return conn.WriteJSON(report)
}
// reportBlock retrieves the current chain head and reports it to the stats server.
func (s *Service) reportBlock(conn *connWrapper, header *types.Header) error {
// Gather the block details from the header or block chain

22
go.mod
View file

@ -31,7 +31,7 @@ require (
github.com/golang-jwt/jwt/v4 v4.5.2
github.com/golang/snappy v1.0.0
github.com/google/gofuzz v1.2.0
github.com/google/uuid v1.3.0
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.4.2
github.com/graph-gophers/graphql-go v1.3.0
github.com/hashicorp/go-bexpr v0.1.10
@ -56,16 +56,19 @@ require (
github.com/rs/cors v1.7.0
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible
github.com/status-im/keycard-go v0.2.0
github.com/stretchr/testify v1.10.0
github.com/stretchr/testify v1.11.1
github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/urfave/cli/v2 v2.27.5
go.opentelemetry.io/otel v1.39.0
go.opentelemetry.io/otel/sdk v1.39.0
go.opentelemetry.io/otel/trace v1.39.0
go.uber.org/automaxprocs v1.5.2
go.uber.org/goleak v1.3.0
golang.org/x/crypto v0.36.0
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
golang.org/x/sync v0.12.0
golang.org/x/sys v0.36.0
golang.org/x/sys v0.39.0
golang.org/x/text v0.23.0
golang.org/x/time v0.9.0
golang.org/x/tools v0.29.0
@ -74,6 +77,13 @@ require (
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
)
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
@ -111,10 +121,12 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
github.com/grafana/pyroscope-go v1.2.7
github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kilic/bls12-381 v0.1.0 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
@ -136,7 +148,7 @@ require (
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect

47
go.sum
View file

@ -136,6 +136,11 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
@ -170,19 +175,23 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac=
github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc=
github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og=
github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
@ -218,8 +227,8 @@ github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4
github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -322,8 +331,8 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@ -335,6 +344,8 @@ github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@ -343,8 +354,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw=
github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
@ -363,6 +374,18 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@ -444,8 +467,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

4
graphql/graphql.go
View file

@ -272,7 +272,7 @@ func (t *Transaction) GasPrice(ctx context.Context) hexutil.Big {
return hexutil.Big{}
}
switch tx.Type() {
case types.DynamicFeeTxType:
case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType:
if block != nil {
if baseFee, _ := block.BaseFeePerGas(ctx); baseFee != nil {
// price = min(gasTipCap + baseFee, gasFeeCap)
@ -1433,7 +1433,7 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria
topics = *args.Filter.Topics
}
// Construct the range filter
filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics)
filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics, 0)
return runFilter(ctx, r, filter)
}

14
internal/debug/flags.go
View file

@ -162,6 +162,11 @@ var Flags = []cli.Flag{
blockprofilerateFlag,
cpuprofileFlag,
traceFlag,
pyroscopeFlag,
pyroscopeServerFlag,
pyroscopeAuthUsernameFlag,
pyroscopeAuthPasswordFlag,
pyroscopeTagsFlag,
}
var (
@ -298,6 +303,14 @@ func Setup(ctx *cli.Context) error {
// It cannot be imported because it will cause a cyclical dependency.
StartPProf(address, !ctx.IsSet("metrics.addr"))
}
// Pyroscope profiling
if ctx.Bool(pyroscopeFlag.Name) {
if err := startPyroscope(ctx); err != nil {
return err
}
}
if len(logFile) > 0 || rotation {
log.Info("Logging configured", context...)
}
@ -321,6 +334,7 @@ func StartPProf(address string, withMetrics bool) {
// Exit stops all running profiles, flushing their output to the
// respective file.
func Exit() {
stopPyroscope()
Handler.StopCPUProfile()
Handler.StopGoTrace()
if logOutputFile != nil {

134
internal/debug/pyroscope.go Normal file
View file

@ -0,0 +1,134 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package debug
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/grafana/pyroscope-go"
"github.com/urfave/cli/v2"
)
var (
pyroscopeFlag = &cli.BoolFlag{
Name: "pyroscope",
Usage: "Enable Pyroscope profiling",
Value: false,
Category: flags.LoggingCategory,
}
pyroscopeServerFlag = &cli.StringFlag{
Name: "pyroscope.server",
Usage: "Pyroscope server URL to push profiling data to",
Value: "http://localhost:4040",
Category: flags.LoggingCategory,
}
pyroscopeAuthUsernameFlag = &cli.StringFlag{
Name: "pyroscope.username",
Usage: "Pyroscope basic authentication username",
Value: "",
Category: flags.LoggingCategory,
}
pyroscopeAuthPasswordFlag = &cli.StringFlag{
Name: "pyroscope.password",
Usage: "Pyroscope basic authentication password",
Value: "",
Category: flags.LoggingCategory,
}
pyroscopeTagsFlag = &cli.StringFlag{
Name: "pyroscope.tags",
Usage: "Comma separated list of key=value tags to add to profiling data",
Value: "",
Category: flags.LoggingCategory,
}
)
// This holds the globally-configured Pyroscope instance.
var pyroscopeProfiler *pyroscope.Profiler
func startPyroscope(ctx *cli.Context) error {
server := ctx.String(pyroscopeServerFlag.Name)
authUsername := ctx.String(pyroscopeAuthUsernameFlag.Name)
authPassword := ctx.String(pyroscopeAuthPasswordFlag.Name)
rawTags := ctx.String(pyroscopeTagsFlag.Name)
tags := make(map[string]string)
for tag := range strings.SplitSeq(rawTags, ",") {
tag = strings.TrimSpace(tag)
if tag == "" {
continue
}
k, v, _ := strings.Cut(tag, "=")
tags[k] = v
}
config := pyroscope.Config{
ApplicationName: "geth",
ServerAddress: server,
BasicAuthUser: authUsername,
BasicAuthPassword: authPassword,
Logger: &pyroscopeLogger{Logger: log.Root()},
Tags: tags,
ProfileTypes: []pyroscope.ProfileType{
// Enabling all profile types
pyroscope.ProfileCPU,
pyroscope.ProfileAllocObjects,
pyroscope.ProfileAllocSpace,
pyroscope.ProfileInuseObjects,
pyroscope.ProfileInuseSpace,
pyroscope.ProfileGoroutines,
pyroscope.ProfileMutexCount,
pyroscope.ProfileMutexDuration,
pyroscope.ProfileBlockCount,
pyroscope.ProfileBlockDuration,
},
}
profiler, err := pyroscope.Start(config)
if err != nil {
return err
}
pyroscopeProfiler = profiler
log.Info("Enabled Pyroscope")
return nil
}
func stopPyroscope() {
if pyroscopeProfiler != nil {
pyroscopeProfiler.Stop()
pyroscopeProfiler = nil
}
}
// Small wrapper for log.Logger to satisfy pyroscope.Logger interface
type pyroscopeLogger struct {
Logger log.Logger
}
func (l *pyroscopeLogger) Infof(format string, v ...any) {
l.Logger.Info(fmt.Sprintf("Pyroscope: "+format, v...))
}
func (l *pyroscopeLogger) Debugf(format string, v ...any) {
l.Logger.Debug(fmt.Sprintf("Pyroscope: "+format, v...))
}
func (l *pyroscopeLogger) Errorf(format string, v ...any) {
l.Logger.Error(fmt.Sprintf("Pyroscope: "+format, v...))
}

33
internal/ethapi/api.go
View file

@ -186,6 +186,15 @@ func NewTxPoolAPI(b Backend) *TxPoolAPI {
return &TxPoolAPI{b}
}
// flattenTxs builds the RPC transaction map keyed by nonce for a set of pool txs.
func flattenTxs(txs types.Transactions, header *types.Header, cfg *params.ChainConfig) map[string]*RPCTransaction {
dump := make(map[string]*RPCTransaction, len(txs))
for _, tx := range txs {
dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, header, cfg)
}
return dump
}
// Content returns the transactions contained within the transaction pool.
func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction {
pending, queue := api.b.TxPoolContent()
@ -196,19 +205,11 @@ func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction
curHeader := api.b.CurrentHeader()
// Flatten the pending transactions
for account, txs := range pending {
dump := make(map[string]*RPCTransaction, len(txs))
for _, tx := range txs {
dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig())
}
content["pending"][account.Hex()] = dump
content["pending"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig())
}
// Flatten the queued transactions
for account, txs := range queue {
dump := make(map[string]*RPCTransaction, len(txs))
for _, tx := range txs {
dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig())
}
content["queued"][account.Hex()] = dump
content["queued"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig())
}
return content
}
@ -220,18 +221,10 @@ func (api *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RP
curHeader := api.b.CurrentHeader()
// Build the pending transactions
dump := make(map[string]*RPCTransaction, len(pending))
for _, tx := range pending {
dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig())
}
content["pending"] = dump
content["pending"] = flattenTxs(pending, curHeader, api.b.ChainConfig())
// Build the queued transactions
dump = make(map[string]*RPCTransaction, len(queue))
for _, tx := range queue {
dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig())
}
content["queued"] = dump
content["queued"] = flattenTxs(queue, curHeader, api.b.ChainConfig())
return content
}

2
internal/flags/helpers.go
View file

@ -40,7 +40,7 @@ func NewApp(usage string) *cli.App {
app.EnableBashCompletion = true
app.Version = version.WithCommit(git.Commit, git.Date)
app.Usage = usage
app.Copyright = "Copyright 2013-2025 The go-ethereum Authors"
app.Copyright = "Copyright 2013-2026 The go-ethereum Authors"
app.Before = func(ctx *cli.Context) error {
MigrateGlobalFlags(ctx)
return nil

104
internal/telemetry/telemetry.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package telemetry
import (
"context"
"fmt"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.38.0"
"go.opentelemetry.io/otel/trace"
)
// Attribute is an alias for attribute.KeyValue.
type Attribute = attribute.KeyValue
// StringAttribute creates an attribute with a string value.
func StringAttribute(key, val string) Attribute {
return attribute.String(key, val)
}
// Int64Attribute creates an attribute with an int64 value.
func Int64Attribute(key string, val int64) Attribute {
return attribute.Int64(key, val)
}
// BoolAttribute creates an attribute with a bool value.
func BoolAttribute(key string, val bool) Attribute {
return attribute.Bool(key, val)
}
// StartSpan creates a SpanKind=INTERNAL span.
func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(error)) {
return StartSpanWithTracer(ctx, otel.Tracer(""), spanName, attributes...)
}
// StartSpanWithTracer requires a tracer to be passed in and creates a SpanKind=INTERNAL span.
func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(error)) {
return startSpan(ctx, tracer, trace.SpanKindInternal, name, attributes...)
}
// RPCInfo contains information about the RPC request.
type RPCInfo struct {
System string
Service string
Method string
RequestID string
}
// StartServerSpan creates a SpanKind=SERVER span at the JSON-RPC boundary.
// The span name is formatted as $rpcSystem.$rpcService/$rpcMethod
// (e.g. "jsonrpc.engine/newPayloadV4") which follows the Open Telemetry
// semantic convensions: https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#span-name.
func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, others ...Attribute) (context.Context, func(error)) {
var (
name = fmt.Sprintf("%s.%s/%s", rpc.System, rpc.Service, rpc.Method)
attributes = append([]Attribute{
semconv.RPCSystemKey.String(rpc.System),
semconv.RPCServiceKey.String(rpc.Service),
semconv.RPCMethodKey.String(rpc.Method),
semconv.RPCJSONRPCRequestID(rpc.RequestID),
},
others...,
)
)
ctx, _, end := startSpan(ctx, tracer, trace.SpanKindServer, name, attributes...)
return ctx, end
}
// startSpan creates a span with the given kind.
func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(error)) {
ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(kind))
if len(attributes) > 0 {
span.SetAttributes(attributes...)
}
return ctx, span, endSpan(span)
}
// endSpan ends the span and handles error recording.
func endSpan(span trace.Span) func(error) {
return func(err error) {
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
}
span.End()
}
}

2
oss-fuzz.sh
View file

@ -64,7 +64,7 @@ function compile_fuzzer() {
go get github.com/holiman/gofuzz-shim/testing
if [[ $SANITIZER == *coverage* ]]; then
coverbuild $path $function $fuzzer $coverpkg
coverbuild $path $function $fuzzer
else
gofuzz-shim --func $function --package $package -f $file -o $fuzzer.a
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer

34
rlp/raw.go
View file

@ -152,6 +152,40 @@ func CountValues(b []byte) (int, error) {
return i, nil
}
// SplitListValues extracts the raw elements from the list RLP-encoding blob.
func SplitListValues(b []byte) ([][]byte, error) {
b, _, err := SplitList(b)
if err != nil {
return nil, err
}
n, err := CountValues(b)
if err != nil {
return nil, err
}
var elements = make([][]byte, 0, n)
for len(b) > 0 {
_, tagsize, size, err := readKind(b)
if err != nil {
return nil, err
}
elements = append(elements, b[:tagsize+size])
b = b[tagsize+size:]
}
return elements, nil
}
// MergeListValues takes a list of raw elements and rlp-encodes them as list.
func MergeListValues(elems [][]byte) ([]byte, error) {
w := NewEncoderBuffer(nil)
offset := w.List()
for _, elem := range elems {
w.Write(elem)
}
w.ListEnd(offset)
return w.ToBytes(), nil
}
func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) {
if len(buf) == 0 {
return 0, 0, 0, io.ErrUnexpectedEOF

266
rlp/raw_test.go
View file

@ -336,3 +336,269 @@ func TestBytesSize(t *testing.T) {
}
}
}
func TestSplitListValues(t *testing.T) {
tests := []struct {
name string
input string // hex-encoded RLP list
want []string // hex-encoded expected elements
wantErr error
}{
{
name: "empty list",
input: "C0",
want: []string{},
},
{
name: "single byte element",
input: "C101",
want: []string{"01"},
},
{
name: "single empty string",
input: "C180",
want: []string{"80"},
},
{
name: "two byte elements",
input: "C20102",
want: []string{"01", "02"},
},
{
name: "three elements",
input: "C3010203",
want: []string{"01", "02", "03"},
},
{
name: "mixed size elements",
input: "C80182020283030303",
want: []string{"01", "820202", "83030303"},
},
{
name: "string elements",
input: "C88363617483646F67",
want: []string{"83636174", "83646F67"}, // cat,dog
},
{
name: "nested list element",
input: "C4C3010203", // [[1,2,3]]
want: []string{"C3010203"}, // [1,2,3]
},
{
name: "multiple nested lists",
input: "C6C20102C20304", // [[1,2],[3,4]]
want: []string{"C20102", "C20304"}, // [1,2], [3,4]
},
{
name: "large list",
input: "C6010203040506",
want: []string{"01", "02", "03", "04", "05", "06"},
},
{
name: "list with empty strings",
input: "C3808080",
want: []string{"80", "80", "80"},
},
// Error cases
{
name: "single byte",
input: "01",
wantErr: ErrExpectedList,
},
{
name: "string",
input: "83636174",
wantErr: ErrExpectedList,
},
{
name: "empty input",
input: "",
wantErr: io.ErrUnexpectedEOF,
},
{
name: "invalid list - value too large",
input: "C60102030405",
wantErr: ErrValueTooLarge,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := SplitListValues(unhex(tt.input))
if !errors.Is(err, tt.wantErr) {
t.Errorf("SplitListValues() error = %v, wantErr %v", err, tt.wantErr)
return
}
if err != nil {
return
}
if len(got) != len(tt.want) {
t.Errorf("SplitListValues() got %d elements, want %d", len(got), len(tt.want))
return
}
for i, elem := range got {
want := unhex(tt.want[i])
if !bytes.Equal(elem, want) {
t.Errorf("SplitListValues() element[%d] = %x, want %x", i, elem, want)
}
}
})
}
}
func TestMergeListValues(t *testing.T) {
tests := []struct {
name string
elems []string // hex-encoded RLP elements
want string // hex-encoded expected result
wantErr error
}{
{
name: "empty list",
elems: []string{},
want: "C0",
},
{
name: "single byte element",
elems: []string{"01"},
want: "C101",
},
{
name: "single empty string",
elems: []string{"80"},
want: "C180",
},
{
name: "two byte elements",
elems: []string{"01", "02"},
want: "C20102",
},
{
name: "three elements",
elems: []string{"01", "02", "03"},
want: "C3010203",
},
{
name: "mixed size elements",
elems: []string{"01", "820202", "83030303"},
want: "C80182020283030303",
},
{
name: "string elements",
elems: []string{"83636174", "83646F67"}, // cat, dog
want: "C88363617483646F67",
},
{
name: "nested list element",
elems: []string{"C20102", "03"}, // [[1, 2], 3]
want: "C4C2010203",
},
{
name: "multiple nested lists",
elems: []string{"C20102", "C3030405"}, // [[1,2],[3,4,5]],
want: "C7C20102C3030405",
},
{
name: "large list",
elems: []string{"01", "02", "03", "04", "05", "06"},
want: "C6010203040506",
},
{
name: "list with empty strings",
elems: []string{"80", "80", "80"},
want: "C3808080",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
elems := make([][]byte, len(tt.elems))
for i, s := range tt.elems {
elems[i] = unhex(s)
}
got, err := MergeListValues(elems)
if !errors.Is(err, tt.wantErr) {
t.Errorf("MergeListValues() error = %v, wantErr %v", err, tt.wantErr)
return
}
if err != nil {
return
}
want := unhex(tt.want)
if !bytes.Equal(got, want) {
t.Errorf("MergeListValues() = %x, want %x", got, want)
}
})
}
}
func TestSplitMergeList(t *testing.T) {
tests := []struct {
name string
input string // hex-encoded RLP list
}{
{
name: "empty list",
input: "C0",
},
{
name: "single byte element",
input: "C101",
},
{
name: "two byte elements",
input: "C20102",
},
{
name: "three elements",
input: "C3010203",
},
{
name: "mixed size elements",
input: "C80182020283030303",
},
{
name: "string elements",
input: "C88363617483646F67", // [cat, dog]
},
{
name: "nested list element",
input: "C4C2010203", // [[1,2],3]
},
{
name: "multiple nested lists",
input: "C6C20102C20304", // [[1,2],[3,4]]
},
{
name: "large list",
input: "C6010203040506", // [1,2,3,4,5,6]
},
{
name: "list with empty strings",
input: "C3808080", // ["", "", ""]
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
original := unhex(tt.input)
// Split the list
elements, err := SplitListValues(original)
if err != nil {
t.Fatalf("SplitListValues() error = %v", err)
}
// Merge back
merged, err := MergeListValues(elements)
if err != nil {
t.Fatalf("MergeListValues() error = %v", err)
}
// The merged result should match the original
if !bytes.Equal(merged, original) {
t.Errorf("Round trip failed: original = %x, merged = %x", original, merged)
}
})
}
}

2
rpc/client.go
View file

@ -119,7 +119,7 @@ func (c *Client) newClientConn(conn ServerCodec) *clientConn {
ctx := context.Background()
ctx = context.WithValue(ctx, clientContextKey{}, c)
ctx = context.WithValue(ctx, peerInfoContextKey{}, conn.peerInfo())
handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize)
handler := newHandler(ctx, conn, c.idgen, c.services, c.batchItemLimit, c.batchResponseMaxSize, nil)
return &clientConn{conn, handler}
}

101
rpc/handler.go
View file

@ -28,7 +28,10 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/internal/telemetry"
"github.com/ethereum/go-ethereum/log"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
// handler handles JSON-RPC messages. There is one handler per connection. Note that
@ -65,6 +68,7 @@ type handler struct {
allowSubscribe bool
batchRequestLimit int
batchResponseMaxSize int
tracerProvider trace.TracerProvider
subLock sync.Mutex
serverSubs map[ID]*Subscription
@ -73,9 +77,10 @@ type handler struct {
type callProc struct {
ctx context.Context
notifiers []*Notifier
isBatch bool
}
func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int) *handler {
func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *serviceRegistry, batchRequestLimit, batchResponseMaxSize int, tracerProvider trace.TracerProvider) *handler {
rootCtx, cancelRoot := context.WithCancel(connCtx)
h := &handler{
reg: reg,
@ -90,6 +95,7 @@ func newHandler(connCtx context.Context, conn jsonWriter, idgen func() ID, reg *
log: log.Root(),
batchRequestLimit: batchRequestLimit,
batchResponseMaxSize: batchResponseMaxSize,
tracerProvider: tracerProvider,
}
if conn.remoteAddr() != "" {
h.log = h.log.New("conn", conn.remoteAddr())
@ -197,6 +203,7 @@ func (h *handler) handleBatch(msgs []*jsonrpcMessage) {
// Process calls on a goroutine because they may block indefinitely:
h.startCallProc(func(cp *callProc) {
cp.isBatch = true
var (
timer *time.Timer
cancel context.CancelFunc
@ -497,40 +504,65 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage
if msg.isSubscribe() {
return h.handleSubscribe(cp, msg)
}
var callb *callback
if msg.isUnsubscribe() {
callb = h.unsubscribeCb
} else {
// Check method name length
if len(msg.Method) > maxMethodNameLength {
return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)})
args, err := parsePositionalArguments(msg.Params, h.unsubscribeCb.argTypes)
if err != nil {
return msg.errorResponse(&invalidParamsError{err.Error()})
}
callb = h.reg.callback(msg.Method)
return h.runMethod(cp.ctx, msg, h.unsubscribeCb, args)
}
// Check method name length
if len(msg.Method) > maxMethodNameLength {
return msg.errorResponse(&invalidRequestError{fmt.Sprintf("method name too long: %d > %d", len(msg.Method), maxMethodNameLength)})
}
callb, service, method := h.reg.callback(msg.Method)
// If the method is not found, return an error.
if callb == nil {
return msg.errorResponse(&methodNotFoundError{method: msg.Method})
}
// Start root span for the request.
var err error
rpcInfo := telemetry.RPCInfo{
System: "jsonrpc",
Service: service,
Method: method,
RequestID: string(msg.ID),
}
attrib := []telemetry.Attribute{
telemetry.BoolAttribute("rpc.batch", cp.isBatch),
}
ctx, spanEnd := telemetry.StartServerSpan(cp.ctx, h.tracer(), rpcInfo, attrib...)
defer spanEnd(err)
// Start tracing span before parsing arguments.
_, _, pSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.parsePositionalArguments")
args, err := parsePositionalArguments(msg.Params, callb.argTypes)
pSpanEnd(err)
if err != nil {
return msg.errorResponse(&invalidParamsError{err.Error()})
}
start := time.Now()
answer := h.runMethod(cp.ctx, msg, callb, args)
// Start tracing span before running the method.
rctx, _, rSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.runMethod")
answer := h.runMethod(rctx, msg, callb, args)
if answer.Error != nil {
err = errors.New(answer.Error.Message)
}
rSpanEnd(err)
// Collect the statistics for RPC calls if metrics is enabled.
// We only care about pure rpc call. Filter out subscription.
if callb != h.unsubscribeCb {
rpcRequestGauge.Inc(1)
if answer.Error != nil {
failedRequestGauge.Inc(1)
} else {
successfulRequestGauge.Inc(1)
}
rpcServingTimer.UpdateSince(start)
updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start))
rpcRequestGauge.Inc(1)
if answer.Error != nil {
failedRequestGauge.Inc(1)
} else {
successfulRequestGauge.Inc(1)
}
rpcServingTimer.UpdateSince(start)
updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start))
return answer
}
@ -568,17 +600,33 @@ func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMes
n := &Notifier{h: h, namespace: namespace}
cp.notifiers = append(cp.notifiers, n)
ctx := context.WithValue(cp.ctx, notifierKey{}, n)
return h.runMethod(ctx, msg, callb, args)
}
// tracer returns the OpenTelemetry Tracer for RPC call tracing.
func (h *handler) tracer() trace.Tracer {
if h.tracerProvider == nil {
// Default to global TracerProvider if none is set.
// Note: If no TracerProvider is set, the default is a no-op TracerProvider.
// See https://pkg.go.dev/go.opentelemetry.io/otel#GetTracerProvider
return otel.Tracer("")
}
return h.tracerProvider.Tracer("")
}
// runMethod runs the Go callback for an RPC method.
func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value) *jsonrpcMessage {
func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *callback, args []reflect.Value, attributes ...telemetry.Attribute) *jsonrpcMessage {
result, err := callb.call(ctx, msg.Method, args)
if err != nil {
return msg.errorResponse(err)
}
return msg.response(result)
_, _, spanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.encodeJSONResponse", attributes...)
response := msg.response(result)
if response.Error != nil {
err = errors.New(response.Error.Message)
}
spanEnd(err)
return response
}
// unsubscribe is the callback function for all *_unsubscribe calls.
@ -612,8 +660,11 @@ type limitedBuffer struct {
}
func (buf *limitedBuffer) Write(data []byte) (int, error) {
avail := max(buf.limit, len(buf.output))
if len(data) < avail {
avail := buf.limit - len(buf.output)
if avail <= 0 {
return 0, errTruncatedOutput
}
if len(data) <= avail {
buf.output = append(buf.output, data...)
return len(data), nil
}

6
rpc/http.go
View file

@ -30,6 +30,9 @@ import (
"strconv"
"sync"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
)
const (
@ -334,6 +337,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = context.WithValue(ctx, peerInfoContextKey{}, connInfo)
// Extract trace context from incoming headers.
ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header))
// All checks passed, create a codec that reads directly from the request body
// until EOF, writes the response to w, and orders the server to process a
// single request.

22
rpc/server.go
View file

@ -25,6 +25,7 @@ import (
"sync/atomic"
"github.com/ethereum/go-ethereum/log"
"go.opentelemetry.io/otel/trace"
)
const MetadataApi = "rpc"
@ -55,15 +56,17 @@ type Server struct {
batchResponseLimit int
httpBodyLimit int
wsReadLimit int64
tracerProvider trace.TracerProvider
}
// NewServer creates a new server instance with no registered handlers.
func NewServer() *Server {
server := &Server{
idgen: randomIDGenerator(),
codecs: make(map[ServerCodec]struct{}),
httpBodyLimit: defaultBodyLimit,
wsReadLimit: wsDefaultReadLimit,
idgen: randomIDGenerator(),
codecs: make(map[ServerCodec]struct{}),
httpBodyLimit: defaultBodyLimit,
wsReadLimit: wsDefaultReadLimit,
tracerProvider: nil,
}
server.run.Store(true)
// Register the default service providing meta information about the RPC service such
@ -129,6 +132,15 @@ func (s *Server) ServeCodec(codec ServerCodec, options CodecOption) {
c.Close()
}
// setTracerProvider configures the OpenTelemetry TracerProvider for RPC call tracing.
// Note: This method (and the TracerProvider field in the Server/Handler struct) is
// primarily intended for testing. In particular, it allows tests to configure an
// isolated TracerProvider without changing the global provider, avoiding
// interference between tests running in parallel.
func (s *Server) setTracerProvider(tp trace.TracerProvider) {
s.tracerProvider = tp
}
func (s *Server) trackCodec(codec ServerCodec) bool {
s.mutex.Lock()
defer s.mutex.Unlock()
@ -156,7 +168,7 @@ func (s *Server) serveSingleRequest(ctx context.Context, codec ServerCodec) {
return
}
h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit)
h := newHandler(ctx, codec, s.idgen, &s.services, s.batchItemLimit, s.batchResponseLimit, s.tracerProvider)
h.allowSubscribe = false
defer h.close(io.EOF, nil)

6
rpc/service.go
View file

@ -92,14 +92,14 @@ func (r *serviceRegistry) registerName(name string, rcvr interface{}) error {
}
// callback returns the callback corresponding to the given RPC method name.
func (r *serviceRegistry) callback(method string) *callback {
func (r *serviceRegistry) callback(method string) (cb *callback, service, methodName string) {
before, after, found := strings.Cut(method, serviceMethodSeparator)
if !found {
return nil
return nil, "", ""
}
r.mu.Lock()
defer r.mu.Unlock()
return r.services[before].callbacks[after]
return r.services[before].callbacks[after], before, after
}
// subscription returns a subscription callback in the given service.

224
rpc/tracing_test.go Normal file
View file

@ -0,0 +1,224 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rpc
import (
"context"
"net/http/httptest"
"testing"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
// attributeMap converts a slice of attributes to a map.
func attributeMap(attrs []attribute.KeyValue) map[string]string {
m := make(map[string]string)
for _, a := range attrs {
switch a.Value.Type() {
case attribute.STRING:
m[string(a.Key)] = a.Value.AsString()
case attribute.BOOL:
if a.Value.AsBool() {
m[string(a.Key)] = "true"
} else {
m[string(a.Key)] = "false"
}
default:
m[string(a.Key)] = a.Value.Emit()
}
}
return m
}
// newTracingServer creates a new server with tracing enabled.
func newTracingServer(t *testing.T) (*Server, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) {
t.Helper()
exporter := tracetest.NewInMemoryExporter()
tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter))
t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
server := newTestServer()
server.setTracerProvider(tp)
t.Cleanup(server.Stop)
return server, tp, exporter
}
// TestTracingHTTP verifies that RPC spans are emitted when processing HTTP requests.
func TestTracingHTTP(t *testing.T) {
// Not parallel: this test modifies the global otel TextMapPropagator.
// Set up a propagator to extract W3C Trace Context headers.
originalPropagator := otel.GetTextMapPropagator()
otel.SetTextMapPropagator(propagation.TraceContext{})
t.Cleanup(func() { otel.SetTextMapPropagator(originalPropagator) })
server, tracer, exporter := newTracingServer(t)
httpsrv := httptest.NewServer(server)
t.Cleanup(httpsrv.Close)
// Define the expected trace and span IDs for context propagation.
const (
traceID = "4bf92f3577b34da6a3ce929d0e0e4736"
parentSpanID = "00f067aa0ba902b7"
traceparent = "00-" + traceID + "-" + parentSpanID + "-01"
)
client, err := DialHTTP(httpsrv.URL)
if err != nil {
t.Fatalf("failed to dial: %v", err)
}
t.Cleanup(client.Close)
// Set trace context headers.
client.SetHeader("traceparent", traceparent)
// Make a successful RPC call.
var result echoResult
if err := client.Call(&result, "test_echo", "hello", 42, &echoArgs{S: "world"}); err != nil {
t.Fatalf("RPC call failed: %v", err)
}
// Flush and verify that we emitted the expected span.
if err := tracer.ForceFlush(context.Background()); err != nil {
t.Fatalf("failed to flush: %v", err)
}
spans := exporter.GetSpans()
if len(spans) == 0 {
t.Fatal("no spans were emitted")
}
var rpcSpan *tracetest.SpanStub
for i := range spans {
if spans[i].Name == "jsonrpc.test/echo" {
rpcSpan = &spans[i]
break
}
}
if rpcSpan == nil {
t.Fatalf("jsonrpc.test/echo span not found")
}
// Verify span attributes.
attrs := attributeMap(rpcSpan.Attributes)
if attrs["rpc.system"] != "jsonrpc" {
t.Errorf("expected rpc.system=jsonrpc, got %v", attrs["rpc.system"])
}
if attrs["rpc.service"] != "test" {
t.Errorf("expected rpc.service=test, got %v", attrs["rpc.service"])
}
if attrs["rpc.method"] != "echo" {
t.Errorf("expected rpc.method=echo, got %v", attrs["rpc.method"])
}
if _, ok := attrs["rpc.jsonrpc.request_id"]; !ok {
t.Errorf("expected rpc.jsonrpc.request_id attribute to be set")
}
// Verify the span's parent matches the traceparent header values.
if got := rpcSpan.Parent.TraceID().String(); got != traceID {
t.Errorf("parent trace ID mismatch: got %s, want %s", got, traceID)
}
if got := rpcSpan.Parent.SpanID().String(); got != parentSpanID {
t.Errorf("parent span ID mismatch: got %s, want %s", got, parentSpanID)
}
if !rpcSpan.Parent.IsRemote() {
t.Error("expected parent span context to be marked as remote")
}
}
// TestTracingBatchHTTP verifies that RPC spans are emitted for batched JSON-RPC calls over HTTP.
func TestTracingBatchHTTP(t *testing.T) {
t.Parallel()
server, tracer, exporter := newTracingServer(t)
httpsrv := httptest.NewServer(server)
t.Cleanup(httpsrv.Close)
client, err := DialHTTP(httpsrv.URL)
if err != nil {
t.Fatalf("failed to dial: %v", err)
}
t.Cleanup(client.Close)
// Make a successful batch RPC call.
batch := []BatchElem{
{
Method: "test_echo",
Args: []any{"hello", 42, &echoArgs{S: "world"}},
Result: new(echoResult),
},
{
Method: "test_echo",
Args: []any{"your", 7, &echoArgs{S: "mom"}},
Result: new(echoResult),
},
}
if err := client.BatchCall(batch); err != nil {
t.Fatalf("batch RPC call failed: %v", err)
}
// Flush and verify we emitted spans for each batch element.
if err := tracer.ForceFlush(context.Background()); err != nil {
t.Fatalf("failed to flush: %v", err)
}
spans := exporter.GetSpans()
if len(spans) == 0 {
t.Fatal("no spans were emitted")
}
var found int
for i := range spans {
if spans[i].Name == "jsonrpc.test/echo" {
attrs := attributeMap(spans[i].Attributes)
if attrs["rpc.system"] == "jsonrpc" &&
attrs["rpc.service"] == "test" &&
attrs["rpc.method"] == "echo" &&
attrs["rpc.batch"] == "true" {
found++
}
}
}
if found != len(batch) {
t.Fatalf("expected %d matching batch spans, got %d", len(batch), found)
}
}
// TestTracingSubscribeUnsubscribe verifies that subscribe and unsubscribe calls
// do not emit any spans.
// Note: This works because client.newClientConn() passes nil as the tracer provider.
func TestTracingSubscribeUnsubscribe(t *testing.T) {
t.Parallel()
server, tracer, exporter := newTracingServer(t)
client := DialInProc(server)
t.Cleanup(client.Close)
// Subscribe to notifications.
sub, err := client.Subscribe(context.Background(), "nftest", make(chan int), "someSubscription", 1, 1)
if err != nil {
t.Fatalf("subscribe failed: %v", err)
}
// Unsubscribe.
sub.Unsubscribe()
// Flush and check that no spans were emitted.
if err := tracer.ForceFlush(context.Background()); err != nil {
t.Fatalf("failed to flush: %v", err)
}
spans := exporter.GetSpans()
if len(spans) != 0 {
t.Errorf("expected no spans for subscribe/unsubscribe, got %d", len(spans))
}
}

2
tests/block_test_util.go
View file

@ -138,7 +138,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t
gspec.Config.TerminalTotalDifficulty = big.NewInt(stdmath.MaxInt64)
}
triedb := triedb.NewDatabase(db, tconf)
gblock, err := gspec.Commit(db, triedb)
gblock, err := gspec.Commit(db, triedb, nil)
if err != nil {
return err
}

14
tests/state_test_util.go
View file

@ -234,6 +234,20 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo
if err != nil {
// Here, an error exists but it was expected.
// We do not check the post state or logs.
// However, if the test defines a post state root, we should check it.
// In case of an error, the state is reverted to the snapshot, so we need to
// recalculate the root.
post := t.json.Post[subtest.Fork][subtest.Index]
if post.Root != (common.UnprefixedHash{}) {
config, _, err := GetChainConfig(subtest.Fork)
if err != nil {
return fmt.Errorf("failed to get chain config: %w", err)
}
root = st.StateDB.IntermediateRoot(config.IsEIP158(new(big.Int).SetUint64(t.json.Env.Number)))
if root != common.Hash(post.Root) {
return fmt.Errorf("post-state root does not match the pre-state root, indicates an error in the test: got %x, want %x", root, post.Root)
}
}
return nil
}
post := t.json.Post[subtest.Fork][subtest.Index]

69
trie/node.go
View file

@ -17,6 +17,7 @@
package trie
import (
"bytes"
"fmt"
"io"
"strings"
@ -242,6 +243,74 @@ func decodeRef(buf []byte) (node, []byte, error) {
}
}
// decodeNodeElements parses the RLP encoding of a trie node and returns all the
// elements in raw byte format.
//
// For full node, it returns a slice of 17 elements;
// For short node, it returns a slice of 2 elements;
func decodeNodeElements(buf []byte) ([][]byte, error) {
if len(buf) == 0 {
return nil, io.ErrUnexpectedEOF
}
return rlp.SplitListValues(buf)
}
// encodeNodeElements encodes the provided node elements into a rlp list.
func encodeNodeElements(elements [][]byte) ([]byte, error) {
if len(elements) != 2 && len(elements) != 17 {
return nil, fmt.Errorf("invalid number of elements: %d", len(elements))
}
return rlp.MergeListValues(elements)
}
// NodeDifference accepts two RLP-encoding nodes and figures out the difference
// between them.
//
// An error is returned if any of the provided blob is nil, or the type of nodes
// are different.
func NodeDifference(oldvalue []byte, newvalue []byte) (int, []int, [][]byte, error) {
oldElems, err := decodeNodeElements(oldvalue)
if err != nil {
return 0, nil, nil, err
}
newElems, err := decodeNodeElements(newvalue)
if err != nil {
return 0, nil, nil, err
}
if len(oldElems) != len(newElems) {
return 0, nil, nil, fmt.Errorf("different node type, old elements: %d, new elements: %d", len(oldElems), len(newElems))
}
var (
indices = make([]int, 0, len(oldElems))
diff = make([][]byte, 0, len(oldElems))
)
for i := 0; i < len(oldElems); i++ {
if !bytes.Equal(oldElems[i], newElems[i]) {
indices = append(indices, i)
diff = append(diff, oldElems[i])
}
}
return len(oldElems), indices, diff, nil
}
// ReassembleNode accepts a RLP-encoding node along with a set of mutations,
// applying the modification diffs according to the indices and re-assemble.
func ReassembleNode(blob []byte, mutations [][][]byte, indices [][]int) ([]byte, error) {
if len(mutations) == 0 && len(indices) == 0 {
return blob, nil
}
elements, err := decodeNodeElements(blob)
if err != nil {
return nil, err
}
for i := 0; i < len(mutations); i++ {
for j, pos := range indices[i] {
elements[pos] = mutations[i][j]
}
}
return encodeNodeElements(elements)
}
// wraps a decoding error with information about the path to the
// invalid child node (for debugging encoding issues).
type decodeError struct {

283
trie/node_test.go
View file

@ -18,9 +18,12 @@ package trie
import (
"bytes"
"math/rand"
"reflect"
"testing"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/testrand"
"github.com/ethereum/go-ethereum/rlp"
)
@ -94,6 +97,286 @@ func TestDecodeFullNode(t *testing.T) {
}
}
func makeTestLeafNode(small bool) []byte {
l := leafNodeEncoder{}
l.Key = hexToCompact(keybytesToHex(testrand.Bytes(10)))
if small {
l.Val = testrand.Bytes(10)
} else {
l.Val = testrand.Bytes(32)
}
buf := rlp.NewEncoderBuffer(nil)
l.encode(buf)
return buf.ToBytes()
}
func makeTestFullNode(small bool) []byte {
n := fullnodeEncoder{}
for i := 0; i < 16; i++ {
switch rand.Intn(3) {
case 0:
// write nil
case 1:
// write hash
n.Children[i] = testrand.Bytes(32)
case 2:
// write embedded node
n.Children[i] = makeTestLeafNode(small)
}
}
n.Children[16] = testrand.Bytes(32) // value
buf := rlp.NewEncoderBuffer(nil)
n.encode(buf)
return buf.ToBytes()
}
func TestEncodeDecodeNodeElements(t *testing.T) {
var nodes [][]byte
nodes = append(nodes, makeTestFullNode(true))
nodes = append(nodes, makeTestFullNode(false))
nodes = append(nodes, makeTestLeafNode(true))
nodes = append(nodes, makeTestLeafNode(false))
for _, blob := range nodes {
elements, err := decodeNodeElements(blob)
if err != nil {
t.Fatalf("Failed to decode node elements: %v", err)
}
enc, err := encodeNodeElements(elements)
if err != nil {
t.Fatalf("Failed to encode node elements: %v", err)
}
if !bytes.Equal(enc, blob) {
t.Fatalf("Unexpected encoded node element, want: %v, got: %v", blob, enc)
}
}
}
func makeTestLeafNodePair() ([]byte, []byte, [][]byte, []int) {
var (
na = leafNodeEncoder{}
nb = leafNodeEncoder{}
)
key := keybytesToHex(testrand.Bytes(10))
na.Key = hexToCompact(key)
nb.Key = hexToCompact(key)
valA := testrand.Bytes(32)
valB := testrand.Bytes(32)
na.Val = valA
nb.Val = valB
bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil)
na.encode(bufa)
nb.encode(bufb)
diff, _ := rlp.EncodeToBytes(valA)
return bufa.ToBytes(), bufb.ToBytes(), [][]byte{diff}, []int{1}
}
func makeTestFullNodePair() ([]byte, []byte, [][]byte, []int) {
var (
na = fullnodeEncoder{}
nb = fullnodeEncoder{}
indices []int
values [][]byte
)
for i := 0; i < 16; i++ {
switch rand.Intn(3) {
case 0:
// write nil
case 1:
// write same
var child []byte
if rand.Intn(2) == 0 {
child = testrand.Bytes(32) // hashnode
} else {
child = makeTestLeafNode(true) // embedded node
}
na.Children[i] = child
nb.Children[i] = child
case 2:
// write different
var (
va []byte
diff []byte
)
rnd := rand.Intn(3)
if rnd == 0 {
va = testrand.Bytes(32) // hashnode
diff, _ = rlp.EncodeToBytes(va)
} else if rnd == 1 {
va = makeTestLeafNode(true) // embedded node
diff = va
} else {
va = nil
diff = rlp.EmptyString
}
vb := testrand.Bytes(32) // hashnode
na.Children[i] = va
nb.Children[i] = vb
indices = append(indices, i)
values = append(values, diff)
}
}
na.Children[16] = nil
nb.Children[16] = nil
bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil)
na.encode(bufa)
nb.encode(bufb)
return bufa.ToBytes(), bufb.ToBytes(), values, indices
}
func TestNodeDifference(t *testing.T) {
type testsuite struct {
old []byte
new []byte
expErr bool
expIndices []int
expValues [][]byte
}
var tests = []testsuite{
// Invalid node data
{
old: nil, new: nil, expErr: true,
},
{
old: testrand.Bytes(32), new: nil, expErr: true,
},
{
old: nil, new: testrand.Bytes(32), expErr: true,
},
{
old: testrand.Bytes(32), new: testrand.Bytes(32), expErr: true,
},
// Different node type
{
old: makeTestLeafNode(true), new: makeTestFullNode(true), expErr: true,
},
}
for range 10 {
va, vb, elements, indices := makeTestLeafNodePair()
tests = append(tests, testsuite{
old: va,
new: vb,
expErr: false,
expIndices: indices,
expValues: elements,
})
}
for range 10 {
va, vb, elements, indices := makeTestFullNodePair()
tests = append(tests, testsuite{
old: va,
new: vb,
expErr: false,
expIndices: indices,
expValues: elements,
})
}
for _, test := range tests {
_, indices, values, err := NodeDifference(test.old, test.new)
if test.expErr && err == nil {
t.Fatal("Expect error, got nil")
}
if !test.expErr && err != nil {
t.Fatalf("Unexpect error, %v", err)
}
if err == nil {
if !reflect.DeepEqual(indices, test.expIndices) {
t.Fatalf("Unexpected indices, want: %v, got: %v", test.expIndices, indices)
}
if !reflect.DeepEqual(values, test.expValues) {
t.Fatalf("Unexpected values, want: %v, got: %v", test.expValues, values)
}
}
}
}
func TestReassembleFullNode(t *testing.T) {
var fn fullnodeEncoder
for i := 0; i < 16; i++ {
if rand.Intn(2) == 0 {
fn.Children[i] = testrand.Bytes(32)
}
}
buf := rlp.NewEncoderBuffer(nil)
fn.encode(buf)
enc := buf.ToBytes()
// Generate a list of diffs
var (
values [][][]byte
indices [][]int
)
for i := 0; i < 10; i++ {
var (
pos = make(map[int]struct{})
poslist []int
valuelist [][]byte
)
for j := 0; j < 3; j++ {
p := rand.Intn(16)
if _, ok := pos[p]; ok {
continue
}
pos[p] = struct{}{}
nh := testrand.Bytes(32)
diff, _ := rlp.EncodeToBytes(nh)
poslist = append(poslist, p)
valuelist = append(valuelist, diff)
fn.Children[p] = nh
}
values = append(values, valuelist)
indices = append(indices, poslist)
}
reassembled, err := ReassembleNode(enc, values, indices)
if err != nil {
t.Fatalf("Failed to re-assemble full node %v", err)
}
buf2 := rlp.NewEncoderBuffer(nil)
fn.encode(buf2)
enc2 := buf2.ToBytes()
if !reflect.DeepEqual(enc2, reassembled) {
t.Fatalf("Unexpeted reassembled node")
}
}
func TestReassembleShortNode(t *testing.T) {
var ln leafNodeEncoder
ln.Key = hexToCompact(keybytesToHex(testrand.Bytes(10)))
ln.Val = testrand.Bytes(10)
buf := rlp.NewEncoderBuffer(nil)
ln.encode(buf)
enc := buf.ToBytes()
// Generate a list of diffs
var (
values [][][]byte
indices [][]int
)
for i := 0; i < 10; i++ {
val := testrand.Bytes(10)
ln.Val = val
diff, _ := rlp.EncodeToBytes(val)
values = append(values, [][]byte{diff})
indices = append(indices, []int{1})
}
reassembled, err := ReassembleNode(enc, values, indices)
if err != nil {
t.Fatalf("Failed to re-assemble full node %v", err)
}
buf2 := rlp.NewEncoderBuffer(nil)
ln.encode(buf2)
enc2 := buf2.ToBytes()
if !reflect.DeepEqual(enc2, reassembled) {
t.Fatalf("Unexpeted reassembled node")
}
}
// goos: darwin
// goarch: arm64
// pkg: github.com/ethereum/go-ethereum/trie

10
triedb/pathdb/buffer.go
View file

@ -132,7 +132,7 @@ func (b *buffer) size() uint64 {
// flush persists the in-memory dirty trie node into the disk if the configured
// memory threshold is reached. Note, all data must be written atomically.
func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) {
func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethdb.AncientWriter, progress []byte, nodesCache, statesCache *fastcache.Cache, id uint64, postFlush func()) {
if b.done != nil {
panic("duplicated flush operation")
}
@ -165,11 +165,9 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.A
//
// This step is crucial to guarantee that the corresponding state history remains
// available for state rollback.
if freezer != nil {
if err := freezer.SyncAncient(); err != nil {
b.flushErr = err
return
}
if err := syncHistory(freezers...); err != nil {
b.flushErr = err
return
}
nodes := b.nodes.write(batch, nodesCache)
accounts, slots := b.states.write(batch, progress, statesCache)

16
triedb/pathdb/config.go
View file

@ -53,6 +53,7 @@ var (
// Defaults contains default settings for Ethereum mainnet.
var Defaults = &Config{
StateHistory: params.FullImmutabilityThreshold,
TrienodeHistory: -1,
EnableStateIndexing: false,
TrieCleanSize: defaultTrieCleanSize,
StateCleanSize: defaultStateCleanSize,
@ -61,14 +62,16 @@ var Defaults = &Config{
// ReadOnly is the config in order to open database in read only mode.
var ReadOnly = &Config{
ReadOnly: true,
TrieCleanSize: defaultTrieCleanSize,
StateCleanSize: defaultStateCleanSize,
ReadOnly: true,
TrienodeHistory: -1,
TrieCleanSize: defaultTrieCleanSize,
StateCleanSize: defaultStateCleanSize,
}
// Config contains the settings for database.
type Config struct {
StateHistory uint64 // Number of recent blocks to maintain state history for, 0: full chain
TrienodeHistory int64 // Number of recent blocks to maintain trienode history for, 0: full chain, negative: disable
EnableStateIndexing bool // Whether to enable state history indexing for external state access
TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data
StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data
@ -108,6 +111,13 @@ func (c *Config) fields() []interface{} {
} else {
list = append(list, "state-history", fmt.Sprintf("last %d blocks", c.StateHistory))
}
if c.TrienodeHistory >= 0 {
if c.TrienodeHistory == 0 {
list = append(list, "trie-history", "entire chain")
} else {
list = append(list, "trie-history", fmt.Sprintf("last %d blocks", c.TrienodeHistory))
}
}
if c.EnableStateIndexing {
list = append(list, "index-history", true)
}

143
triedb/pathdb/database.go
View file

@ -137,6 +137,9 @@ type Database struct {
stateFreezer ethdb.ResettableAncientStore // Freezer for storing state histories, nil possible in tests
stateIndexer *historyIndexer // History indexer historical state data, nil possible
trienodeFreezer ethdb.ResettableAncientStore // Freezer for storing trienode histories, nil possible in tests
trienodeIndexer *historyIndexer // History indexer for historical trienode data
lock sync.RWMutex // Lock to prevent mutations from happening at the same time
}
@ -169,11 +172,14 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
// and in-memory layer journal.
db.tree = newLayerTree(db.loadLayers())
// Repair the state history, which might not be aligned with the state
// in the key-value store due to an unclean shutdown.
if err := db.repairHistory(); err != nil {
log.Crit("Failed to repair state history", "err", err)
// Repair the history, which might not be aligned with the persistent
// state in the key-value store due to an unclean shutdown.
states, trienodes, err := repairHistory(db.diskdb, isVerkle, db.config.ReadOnly, db.tree.bottom().stateID(), db.config.TrienodeHistory >= 0)
if err != nil {
log.Crit("Failed to repair history", "err", err)
}
db.stateFreezer, db.trienodeFreezer = states, trienodes
// Disable database in case node is still in the initial state sync stage.
if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly {
if err := db.Disable(); err != nil {
@ -187,11 +193,8 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
if err := db.setStateGenerator(); err != nil {
log.Crit("Failed to setup the generator", "err", err)
}
// TODO (rjl493456442) disable the background indexing in read-only mode
if db.stateFreezer != nil && db.config.EnableStateIndexing {
db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory)
log.Info("Enabled state history indexing")
}
db.setHistoryIndexer()
fields := config.fields()
if db.isVerkle {
fields = append(fields, "verkle", true)
@ -200,59 +203,28 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
return db
}
// repairHistory truncates leftover state history objects, which may occur due
// to an unclean shutdown or other unexpected reasons.
func (db *Database) repairHistory() error {
// Open the freezer for state history. This mechanism ensures that
// only one database instance can be opened at a time to prevent
// accidental mutation.
ancient, err := db.diskdb.AncientDatadir()
if err != nil {
// TODO error out if ancient store is disabled. A tons of unit tests
// disable the ancient store thus the error here will immediately fail
// all of them. Fix the tests first.
return nil
// setHistoryIndexer initializes the indexers for both state history and
// trienode history if available. Note that this function may be called while
// existing indexers are still running, so they must be closed beforehand.
func (db *Database) setHistoryIndexer() {
// TODO (rjl493456442) disable the background indexing in read-only mode
if !db.config.EnableStateIndexing {
return
}
freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly)
if err != nil {
log.Crit("Failed to open state history freezer", "err", err)
}
db.stateFreezer = freezer
// Reset the entire state histories if the trie database is not initialized
// yet. This action is necessary because these state histories are not
// expected to exist without an initialized trie database.
id := db.tree.bottom().stateID()
if id == 0 {
frozen, err := db.stateFreezer.Ancients()
if err != nil {
log.Crit("Failed to retrieve head of state history", "err", err)
if db.stateFreezer != nil {
if db.stateIndexer != nil {
db.stateIndexer.close()
}
if frozen != 0 {
// Purge all state history indexing data first
batch := db.diskdb.NewBatch()
rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndexes(batch)
if err := batch.Write(); err != nil {
log.Crit("Failed to purge state history index", "err", err)
}
if err := db.stateFreezer.Reset(); err != nil {
log.Crit("Failed to reset state histories", "err", err)
}
log.Info("Truncated extraneous state history")
db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory)
log.Info("Enabled state history indexing")
}
if db.trienodeFreezer != nil {
if db.trienodeIndexer != nil {
db.trienodeIndexer.close()
}
return nil
db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory)
log.Info("Enabled trienode history indexing")
}
// Truncate the extra state histories above in freezer in case it's not
// aligned with the disk layer. It might happen after a unclean shutdown.
pruned, err := truncateFromHead(db.stateFreezer, typeStateHistory, id)
if err != nil {
log.Crit("Failed to truncate extra state histories", "err", err)
}
if pruned != 0 {
log.Warn("Truncated extra state histories", "number", pruned)
}
return nil
}
// setStateGenerator loads the state generation progress marker and potentially
@ -333,8 +305,13 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6
if err := db.modifyAllowed(); err != nil {
return err
}
// TODO(rjl493456442) tracking the origins in the following PRs.
if err := db.tree.add(root, parentRoot, block, NewNodeSetWithOrigin(nodes.Nodes(), nil), states); err != nil {
var nodesWithOrigins *nodeSetWithOrigin
if db.config.TrienodeHistory >= 0 {
nodesWithOrigins = NewNodeSetWithOrigin(nodes.NodeAndOrigins())
} else {
nodesWithOrigins = NewNodeSetWithOrigin(nodes.Nodes(), nil)
}
if err := db.tree.add(root, parentRoot, block, nodesWithOrigins, states); err != nil {
return err
}
// Keep 128 diff layers in the memory, persistent layer is 129th.
@ -422,18 +399,9 @@ func (db *Database) Enable(root common.Hash) error {
// all root->id mappings should be removed as well. Since
// mappings can be huge and might take a while to clear
// them, just leave them in disk and wait for overwriting.
if db.stateFreezer != nil {
// Purge all state history indexing data first
batch.Reset()
rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndexes(batch)
if err := batch.Write(); err != nil {
return err
}
if err := db.stateFreezer.Reset(); err != nil {
return err
}
}
purgeHistory(db.stateFreezer, db.diskdb, typeStateHistory)
purgeHistory(db.trienodeFreezer, db.diskdb, typeTrienodeHistory)
// Re-enable the database as the final step.
db.waitSync = false
rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished)
@ -446,11 +414,8 @@ func (db *Database) Enable(root common.Hash) error {
// To ensure the history indexer always matches the current state, we must:
// 1. Close any existing indexer
// 2. Re-initialize the indexer so it starts indexing from the new state root.
if db.stateIndexer != nil && db.stateFreezer != nil && db.config.EnableStateIndexing {
db.stateIndexer.close()
db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory)
log.Info("Re-enabled state history indexing")
}
db.setHistoryIndexer()
log.Info("Rebuilt trie database", "root", root)
return nil
}
@ -506,6 +471,12 @@ func (db *Database) Recover(root common.Hash) error {
if err != nil {
return err
}
if db.trienodeFreezer != nil {
_, err = truncateFromHead(db.trienodeFreezer, typeTrienodeHistory, dl.stateID())
if err != nil {
return err
}
}
log.Debug("Recovered state", "root", root, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
@ -566,11 +537,21 @@ func (db *Database) Close() error {
if db.stateIndexer != nil {
db.stateIndexer.close()
}
// Close the attached state history freezer.
if db.stateFreezer == nil {
return nil
if db.trienodeIndexer != nil {
db.trienodeIndexer.close()
}
return db.stateFreezer.Close()
// Close the attached state history freezer.
if db.stateFreezer != nil {
if err := db.stateFreezer.Close(); err != nil {
return err
}
}
if db.trienodeFreezer != nil {
if err := db.trienodeFreezer.Close(); err != nil {
return err
}
}
return nil
}
// Size returns the current storage size of the memory cache in front of the

4
triedb/pathdb/database_test.go
View file

@ -950,7 +950,7 @@ func TestDatabaseIndexRecovery(t *testing.T) {
var (
dIndex int
roots = env.roots
hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer)
hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer)
)
for i, root := range roots {
if root == dRoot {
@ -1011,7 +1011,7 @@ func TestDatabaseIndexRecovery(t *testing.T) {
// Ensure the truncated state histories become accessible
bRoot = env.db.tree.bottom().rootHash()
hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer)
hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer)
for i, root := range roots {
if root == bRoot {
break

74
triedb/pathdb/disklayer.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
@ -323,36 +324,60 @@ func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes *no
return newDiffLayer(dl, root, id, block, nodes, states)
}
// writeStateHistory stores the state history and indexes if indexing is
// writeHistory stores the specified history and indexes if indexing is
// permitted.
//
// What's more, this function also returns a flag indicating whether the
// buffer flushing is required, ensuring the persistent state ID is always
// greater than or equal to the first history ID.
func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) {
// Short circuit if state history is not permitted
if dl.db.stateFreezer == nil {
func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error) {
var (
limit uint64
freezer ethdb.AncientStore
indexer *historyIndexer
writeFunc func(writer ethdb.AncientWriter, dl *diffLayer) error
)
switch typ {
case typeStateHistory:
freezer = dl.db.stateFreezer
indexer = dl.db.stateIndexer
writeFunc = writeStateHistory
limit = dl.db.config.StateHistory
case typeTrienodeHistory:
freezer = dl.db.trienodeFreezer
indexer = dl.db.trienodeIndexer
writeFunc = writeTrienodeHistory
// Skip the history commit if the trienode history is not permitted
if dl.db.config.TrienodeHistory < 0 {
return false, nil
}
limit = uint64(dl.db.config.TrienodeHistory)
default:
panic(fmt.Sprintf("unknown history type: %v", typ))
}
// Short circuit if the history freezer is nil
if freezer == nil {
return false, nil
}
// Bail out with an error if writing the state history fails.
// This can happen, for example, if the device is full.
err := writeStateHistory(dl.db.stateFreezer, diff)
err := writeFunc(freezer, diff)
if err != nil {
return false, err
}
// Notify the state history indexer for newly created history
if dl.db.stateIndexer != nil {
if err := dl.db.stateIndexer.extend(diff.stateID()); err != nil {
// Notify the history indexer for newly created history
if indexer != nil {
if err := indexer.extend(diff.stateID()); err != nil {
return false, err
}
}
// Determine if the persisted history object has exceeded the
// configured limitation.
limit := dl.db.config.StateHistory
if limit == 0 {
return false, nil
}
tail, err := dl.db.stateFreezer.Tail()
tail, err := freezer.Tail()
if err != nil {
return false, err
} // firstID = tail+1
@ -375,14 +400,14 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) {
// These measures ensure the persisted state ID always remains greater
// than or equal to the first history ID.
if persistentID := rawdb.ReadPersistentStateID(dl.db.diskdb); persistentID < newFirst {
log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit)
log.Debug("Skip tail truncation", "type", typ, "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit)
return true, nil
}
pruned, err := truncateFromTail(dl.db.stateFreezer, typeStateHistory, newFirst-1)
pruned, err := truncateFromTail(freezer, typ, newFirst-1)
if err != nil {
return false, err
}
log.Debug("Pruned state history", "items", pruned, "tailid", newFirst)
log.Debug("Pruned history", "type", typ, "items", pruned, "tailid", newFirst)
return false, nil
}
@ -396,10 +421,22 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// Construct and store the state history first. If crash happens after storing
// the state history but without flushing the corresponding states(journal),
// the stored state history will be truncated from head in the next restart.
flush, err := dl.writeStateHistory(bottom)
flushA, err := dl.writeHistory(typeStateHistory, bottom)
if err != nil {
return nil, err
}
// Construct and store the trienode history first. If crash happens after
// storing the trienode history but without flushing the corresponding
// states(journal), the stored trienode history will be truncated from head
// in the next restart.
flushB, err := dl.writeHistory(typeTrienodeHistory, bottom)
if err != nil {
return nil, err
}
// Since the state history and trienode history may be configured with different
// lengths, the buffer will be flushed once either of them meets its threshold.
flush := flushA || flushB
// Mark the diskLayer as stale before applying any mutations on top.
dl.stale = true
@ -448,7 +485,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
// Freeze the live buffer and schedule background flushing
dl.frozen = combined
dl.frozen.flush(bottom.root, dl.db.diskdb, dl.db.stateFreezer, progress, dl.nodes, dl.states, bottom.stateID(), func() {
dl.frozen.flush(bottom.root, dl.db.diskdb, []ethdb.AncientWriter{dl.db.stateFreezer, dl.db.trienodeFreezer}, progress, dl.nodes, dl.states, bottom.stateID(), func() {
// Resume the background generation if it's not completed yet.
// The generator is assumed to be available if the progress is
// not nil.
@ -504,12 +541,17 @@ func (dl *diskLayer) revert(h *stateHistory) (*diskLayer, error) {
dl.stale = true
// Unindex the corresponding state history
// Unindex the corresponding history
if dl.db.stateIndexer != nil {
if err := dl.db.stateIndexer.shorten(dl.id); err != nil {
return nil, err
}
}
if dl.db.trienodeIndexer != nil {
if err := dl.db.trienodeIndexer.shorten(dl.id); err != nil {
return nil, err
}
}
// State change may be applied to node buffer, or the persistent
// state, depends on if node buffer is empty or not. If the node
// buffer is not empty, it means that the state transition that

210
triedb/pathdb/history.go
View file

@ -22,6 +22,7 @@ import (
"iter"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
@ -121,6 +122,20 @@ func (ident stateIdent) String() string {
return ident.addressHash.Hex() + ident.path
}
func (ident stateIdent) bloomSize() int {
if ident.typ == typeAccount {
return 0
}
if ident.typ == typeStorage {
return 0
}
scheme := accountIndexScheme
if ident.addressHash != (common.Hash{}) {
scheme = storageIndexScheme
}
return scheme.getBitmapSize(len(ident.path))
}
// newAccountIdent constructs a state identifier for an account.
func newAccountIdent(addressHash common.Hash) stateIdent {
return stateIdent{
@ -143,6 +158,8 @@ func newStorageIdent(addressHash common.Hash, storageHash common.Hash) stateIden
// newTrienodeIdent constructs a state identifier for a trie node.
// The address denotes the address hash of the associated account;
// the path denotes the path of the node within the trie;
//
// nolint:unused
func newTrienodeIdent(addressHash common.Hash, path string) stateIdent {
return stateIdent{
typ: typeTrienode,
@ -180,17 +197,62 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora
}
}
// newTrienodeIdentQuery constructs a state identifier for a trie node.
// the addressHash denotes the address hash of the associated account;
// the path denotes the path of the node within the trie;
//
// nolint:unused
func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery {
return stateIdentQuery{
stateIdent: newTrienodeIdent(addrHash, string(path)),
// indexElem defines the element for indexing.
type indexElem interface {
key() stateIdent
ext() []uint16
}
type accountIndexElem struct {
addressHash common.Hash
}
func (a accountIndexElem) key() stateIdent {
return stateIdent{
typ: typeAccount,
addressHash: a.addressHash,
}
}
func (a accountIndexElem) ext() []uint16 {
return nil
}
type storageIndexElem struct {
addressHash common.Hash
storageHash common.Hash
}
func (a storageIndexElem) key() stateIdent {
return stateIdent{
typ: typeStorage,
addressHash: a.addressHash,
storageHash: a.storageHash,
}
}
func (a storageIndexElem) ext() []uint16 {
return nil
}
type trienodeIndexElem struct {
owner common.Hash
path string
data []uint16
}
func (a trienodeIndexElem) key() stateIdent {
return stateIdent{
typ: typeTrienode,
addressHash: a.owner,
path: a.path,
}
}
func (a trienodeIndexElem) ext() []uint16 {
return a.data
}
// history defines the interface of historical data, shared by stateHistory
// and trienodeHistory.
type history interface {
@ -198,7 +260,7 @@ type history interface {
typ() historyType
// forEach returns an iterator to traverse the state entries in the history.
forEach() iter.Seq[stateIdent]
forEach() iter.Seq[indexElem]
}
var (
@ -262,3 +324,133 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) (
// Associated root->id mappings are left in the database.
return int(ntail - otail), nil
}
// purgeHistory resets the history and also purges the associated index data.
func purgeHistory(store ethdb.ResettableAncientStore, disk ethdb.KeyValueStore, typ historyType) {
if store == nil {
return
}
frozen, err := store.Ancients()
if err != nil {
log.Crit("Failed to retrieve head of history", "type", typ, "err", err)
}
if frozen == 0 {
return
}
// Purge all state history indexing data first
batch := disk.NewBatch()
if typ == typeStateHistory {
rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndexes(batch)
} else {
rawdb.DeleteTrienodeHistoryIndexMetadata(batch)
rawdb.DeleteTrienodeHistoryIndexes(batch)
}
if err := batch.Write(); err != nil {
log.Crit("Failed to purge history index", "type", typ, "err", err)
}
if err := store.Reset(); err != nil {
log.Crit("Failed to reset history", "type", typ, "err", err)
}
log.Info("Truncated extraneous history", "type", typ)
}
// syncHistory explicitly sync the provided history stores.
func syncHistory(stores ...ethdb.AncientWriter) error {
for _, store := range stores {
if store == nil {
continue
}
if err := store.SyncAncient(); err != nil {
return err
}
}
return nil
}
// repairHistory truncates any leftover history objects in either the state
// history or the trienode history, which may occur due to an unclean shutdown
// or other unexpected events.
//
// Additionally, this mechanism ensures that the state history and trienode
// history remain aligned. Since the trienode history is optional and not
// required by regular users, a gap between the trienode history and the
// persistent state may appear if the trienode history was disabled during the
// previous run. This process detects and resolves such gaps, preventing
// unexpected panics.
func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint64, enableTrienode bool) (ethdb.ResettableAncientStore, ethdb.ResettableAncientStore, error) {
ancient, err := db.AncientDatadir()
if err != nil {
// TODO error out if ancient store is disabled. A tons of unit tests
// disable the ancient store thus the error here will immediately fail
// all of them. Fix the tests first.
return nil, nil, nil
}
// State history is mandatory as it is the key component that ensures
// resilience to deep reorgs.
states, err := rawdb.NewStateFreezer(ancient, isVerkle, readOnly)
if err != nil {
log.Crit("Failed to open state history freezer", "err", err)
}
// Trienode history is optional and only required for building archive
// node with state proofs.
var trienodes ethdb.ResettableAncientStore
if enableTrienode {
trienodes, err = rawdb.NewTrienodeFreezer(ancient, isVerkle, readOnly)
if err != nil {
log.Crit("Failed to open trienode history freezer", "err", err)
}
}
// Reset the both histories if the trie database is not initialized yet.
// This action is necessary because these histories are not expected
// to exist without an initialized trie database.
if stateID == 0 {
purgeHistory(states, db, typeStateHistory)
purgeHistory(trienodes, db, typeTrienodeHistory)
return states, trienodes, nil
}
// Truncate excessive history entries in either the state history or
// the trienode history, ensuring both histories remain aligned with
// the state.
head, err := states.Ancients()
if err != nil {
return nil, nil, err
}
if stateID > head {
return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, head)
}
if trienodes != nil {
th, err := trienodes.Ancients()
if err != nil {
return nil, nil, err
}
if stateID > th {
return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, th)
}
if th != head {
log.Info("Histories are not aligned with each other", "state", head, "trienode", th)
head = min(head, th)
}
}
head = min(head, stateID)
// Truncate the extra history elements above in freezer in case it's not
// aligned with the state. It might happen after an unclean shutdown.
truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) {
if store == nil {
return
}
pruned, err := truncateFromHead(store, typ, nhead)
if err != nil {
log.Crit("Failed to truncate extra histories", "typ", typ, "err", err)
}
if pruned != 0 {
log.Warn("Truncated extra histories", "typ", typ, "number", pruned)
}
}
truncate(states, typeStateHistory, head)
truncate(trienodes, typeTrienodeHistory, head)
return states, trienodes, nil
}

174
triedb/pathdb/history_index.go
View file

@ -25,22 +25,28 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
)
// parseIndex parses the index data with the supplied byte stream. The index data
// is a list of fixed-sized metadata. Empty metadata is regarded as invalid.
func parseIndex(blob []byte) ([]*indexBlockDesc, error) {
// parseIndex parses the index data from the provided byte stream. The index data
// is a sequence of fixed-size metadata entries, and any empty metadata entry is
// considered invalid.
//
// Each metadata entry consists of two components: the indexBlockDesc and an
// optional extension bitmap. The bitmap length may vary across different categories,
// but must remain consistent within the same category.
func parseIndex(blob []byte, bitmapSize int) ([]*indexBlockDesc, error) {
if len(blob) == 0 {
return nil, errors.New("empty state history index")
}
if len(blob)%indexBlockDescSize != 0 {
return nil, fmt.Errorf("corrupted state index, len: %d", len(blob))
size := indexBlockDescSize + bitmapSize
if len(blob)%size != 0 {
return nil, fmt.Errorf("corrupted state index, len: %d, bitmap size: %d", len(blob), bitmapSize)
}
var (
lastID uint32
descList []*indexBlockDesc
)
for i := 0; i < len(blob)/indexBlockDescSize; i++ {
for i := 0; i < len(blob)/size; i++ {
var desc indexBlockDesc
desc.decode(blob[i*indexBlockDescSize : (i+1)*indexBlockDescSize])
desc.decode(blob[i*size : (i+1)*size])
if desc.empty() {
return nil, errors.New("empty state history index block")
}
@ -69,33 +75,35 @@ func parseIndex(blob []byte) ([]*indexBlockDesc, error) {
// indexReader is the structure to look up the state history index records
// associated with the specific state element.
type indexReader struct {
db ethdb.KeyValueReader
descList []*indexBlockDesc
readers map[uint32]*blockReader
state stateIdent
db ethdb.KeyValueReader
descList []*indexBlockDesc
readers map[uint32]*blockReader
state stateIdent
bitmapSize int
}
// loadIndexData loads the index data associated with the specified state.
func loadIndexData(db ethdb.KeyValueReader, state stateIdent) ([]*indexBlockDesc, error) {
func loadIndexData(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) ([]*indexBlockDesc, error) {
blob := readStateIndex(state, db)
if len(blob) == 0 {
return nil, nil
}
return parseIndex(blob)
return parseIndex(blob, bitmapSize)
}
// newIndexReader constructs a index reader for the specified state. Reader with
// empty data is allowed.
func newIndexReader(db ethdb.KeyValueReader, state stateIdent) (*indexReader, error) {
descList, err := loadIndexData(db, state)
func newIndexReader(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) (*indexReader, error) {
descList, err := loadIndexData(db, state, bitmapSize)
if err != nil {
return nil, err
}
return &indexReader{
descList: descList,
readers: make(map[uint32]*blockReader),
db: db,
state: state,
descList: descList,
readers: make(map[uint32]*blockReader),
db: db,
state: state,
bitmapSize: bitmapSize,
}, nil
}
@ -106,11 +114,9 @@ func (r *indexReader) refresh() error {
// may have been modified by additional elements written to the disk.
if len(r.descList) != 0 {
last := r.descList[len(r.descList)-1]
if !last.full() {
delete(r.readers, last.id)
}
delete(r.readers, last.id)
}
descList, err := loadIndexData(r.db, r.state)
descList, err := loadIndexData(r.db, r.state, r.bitmapSize)
if err != nil {
return err
}
@ -118,26 +124,10 @@ func (r *indexReader) refresh() error {
return nil
}
// newIterator creates an iterator for traversing the index entries.
func (r *indexReader) newIterator() *indexIterator {
return newIndexIterator(r.descList, func(id uint32) (*blockReader, error) {
br, ok := r.readers[id]
if !ok {
var err error
br, err = newBlockReader(readStateIndexBlock(r.state, r.db, id))
if err != nil {
return nil, err
}
r.readers[id] = br
}
return br, nil
})
}
// readGreaterThan locates the first element that is greater than the specified
// id. If no such element is found, MaxUint64 is returned.
func (r *indexReader) readGreaterThan(id uint64) (uint64, error) {
it := r.newIterator()
it := r.newIterator(nil)
found := it.SeekGT(id)
if err := it.Error(); err != nil {
return 0, err
@ -155,31 +145,33 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) {
// history ids) is stored in these second-layer index blocks, which are size
// limited.
type indexWriter struct {
descList []*indexBlockDesc // The list of index block descriptions
bw *blockWriter // The live index block writer
frozen []*blockWriter // The finalized index block writers, waiting for flush
lastID uint64 // The ID of the latest tracked history
state stateIdent
db ethdb.KeyValueReader
descList []*indexBlockDesc // The list of index block descriptions
bw *blockWriter // The live index block writer
frozen []*blockWriter // The finalized index block writers, waiting for flush
lastID uint64 // The ID of the latest tracked history
state stateIdent // The identifier of the state being indexed
bitmapSize int // The size of optional extension bitmap
db ethdb.KeyValueReader
}
// newIndexWriter constructs the index writer for the specified state. Additionally,
// it takes an integer as the limit and prunes all existing elements above that ID.
// It's essential as the recovery mechanism after unclean shutdown during the history
// indexing.
func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*indexWriter, error) {
func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexWriter, error) {
blob := readStateIndex(state, db)
if len(blob) == 0 {
desc := newIndexBlockDesc(0)
bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */)
desc := newIndexBlockDesc(0, bitmapSize)
bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0)
return &indexWriter{
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
db: db,
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
db: db,
bitmapSize: bitmapSize,
}, nil
}
descList, err := parseIndex(blob)
descList, err := parseIndex(blob, bitmapSize)
if err != nil {
return nil, err
}
@ -197,30 +189,31 @@ func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*i
// Construct the writer for the last block. All elements in this block
// that exceed the limit will be truncated.
bw, err := newBlockWriter(indexBlock, lastDesc, limit)
bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0)
if err != nil {
return nil, err
}
return &indexWriter{
descList: descList,
lastID: bw.last(),
bw: bw,
state: state,
db: db,
descList: descList,
lastID: bw.last(),
bw: bw,
state: state,
db: db,
bitmapSize: bitmapSize,
}, nil
}
// append adds the new element into the index writer.
func (w *indexWriter) append(id uint64) error {
func (w *indexWriter) append(id uint64, ext []uint16) error {
if id <= w.lastID {
return fmt.Errorf("append element out of order, last: %d, this: %d", w.lastID, id)
}
if w.bw.full() {
if w.bw.estimateFull(ext) {
if err := w.rotate(); err != nil {
return err
}
}
if err := w.bw.append(id); err != nil {
if err := w.bw.append(id, ext); err != nil {
return err
}
w.lastID = id
@ -233,10 +226,10 @@ func (w *indexWriter) append(id uint64) error {
func (w *indexWriter) rotate() error {
var (
err error
desc = newIndexBlockDesc(w.bw.desc.id + 1)
desc = newIndexBlockDesc(w.bw.desc.id+1, w.bitmapSize)
)
w.frozen = append(w.frozen, w.bw)
w.bw, err = newBlockWriter(nil, desc, 0 /* useless if the block is empty */)
w.bw, err = newBlockWriter(nil, desc, 0 /* useless if the block is empty */, w.bitmapSize != 0)
if err != nil {
return err
}
@ -268,7 +261,8 @@ func (w *indexWriter) finish(batch ethdb.Batch) {
}
w.frozen = nil // release all the frozen writers
buf := make([]byte, 0, indexBlockDescSize*len(descList))
size := indexBlockDescSize + w.bitmapSize
buf := make([]byte, 0, size*len(descList))
for _, desc := range descList {
buf = append(buf, desc.encode()...)
}
@ -277,30 +271,32 @@ func (w *indexWriter) finish(batch ethdb.Batch) {
// indexDeleter is responsible for deleting index data for a specific state.
type indexDeleter struct {
descList []*indexBlockDesc // The list of index block descriptions
bw *blockWriter // The live index block writer
dropped []uint32 // The list of index block id waiting for deleting
lastID uint64 // The ID of the latest tracked history
state stateIdent
db ethdb.KeyValueReader
descList []*indexBlockDesc // The list of index block descriptions
bw *blockWriter // The live index block writer
dropped []uint32 // The list of index block id waiting for deleting
lastID uint64 // The ID of the latest tracked history
state stateIdent // The identifier of the state being indexed
bitmapSize int // The size of optional extension bitmap
db ethdb.KeyValueReader
}
// newIndexDeleter constructs the index deleter for the specified state.
func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*indexDeleter, error) {
func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexDeleter, error) {
blob := readStateIndex(state, db)
if len(blob) == 0 {
// TODO(rjl493456442) we can probably return an error here,
// deleter with no data is meaningless.
desc := newIndexBlockDesc(0)
bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */)
desc := newIndexBlockDesc(0, bitmapSize)
bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0)
return &indexDeleter{
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
db: db,
descList: []*indexBlockDesc{desc},
bw: bw,
state: state,
bitmapSize: bitmapSize,
db: db,
}, nil
}
descList, err := parseIndex(blob)
descList, err := parseIndex(blob, bitmapSize)
if err != nil {
return nil, err
}
@ -318,16 +314,17 @@ func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*
// Construct the writer for the last block. All elements in this block
// that exceed the limit will be truncated.
bw, err := newBlockWriter(indexBlock, lastDesc, limit)
bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0)
if err != nil {
return nil, err
}
return &indexDeleter{
descList: descList,
lastID: bw.last(),
bw: bw,
state: state,
db: db,
descList: descList,
lastID: bw.last(),
bw: bw,
state: state,
bitmapSize: bitmapSize,
db: db,
}, nil
}
@ -364,7 +361,7 @@ func (d *indexDeleter) pop(id uint64) error {
// Open the previous block writer for deleting
lastDesc := d.descList[len(d.descList)-1]
indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id)
bw, err := newBlockWriter(indexBlock, lastDesc, lastDesc.max)
bw, err := newBlockWriter(indexBlock, lastDesc, lastDesc.max, d.bitmapSize != 0)
if err != nil {
return err
}
@ -390,7 +387,8 @@ func (d *indexDeleter) finish(batch ethdb.Batch) {
if d.empty() {
deleteStateIndex(d.state, batch)
} else {
buf := make([]byte, 0, indexBlockDescSize*len(d.descList))
size := indexBlockDescSize + d.bitmapSize
buf := make([]byte, 0, size*len(d.descList))
for _, desc := range d.descList {
buf = append(buf, desc.encode()...)
}

220
triedb/pathdb/history_index_block.go
View file

@ -17,6 +17,7 @@
package pathdb
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
@ -26,23 +27,27 @@ import (
)
const (
indexBlockDescSize = 14 // The size of index block descriptor
indexBlockEntriesCap = 4096 // The maximum number of entries can be grouped in a block
indexBlockRestartLen = 256 // The restart interval length of index block
historyIndexBatch = 8 * 1024 * 1024 // The number of state history indexes for constructing or deleting as batch
indexBlockDescSize = 14 // The size of index block descriptor
indexBlockMaxSize = 4096 // The maximum size of a single index block
indexBlockRestartLen = 256 // The restart interval length of index block
)
// indexBlockDesc represents a descriptor for an index block, which contains a
// list of state mutation records associated with a specific state (either an
// account or a storage slot).
type indexBlockDesc struct {
max uint64 // The maximum state ID retained within the block
entries uint16 // The number of state mutation records retained within the block
id uint32 // The id of the index block
max uint64 // The maximum state ID retained within the block
entries uint16 // The number of state mutation records retained within the block
id uint32 // The id of the index block
extBitmap []byte // Optional fixed-size bitmap for the included extension elements
}
func newIndexBlockDesc(id uint32) *indexBlockDesc {
return &indexBlockDesc{id: id}
func newIndexBlockDesc(id uint32, bitmapSize int) *indexBlockDesc {
var bitmap []byte
if bitmapSize > 0 {
bitmap = make([]byte, bitmapSize)
}
return &indexBlockDesc{id: id, extBitmap: bitmap}
}
// empty indicates whether the block is empty with no element retained.
@ -50,26 +55,33 @@ func (d *indexBlockDesc) empty() bool {
return d.entries == 0
}
// full indicates whether the number of elements in the block exceeds the
// preconfigured limit.
func (d *indexBlockDesc) full() bool {
return d.entries >= indexBlockEntriesCap
}
// encode packs index block descriptor into byte stream.
func (d *indexBlockDesc) encode() []byte {
var buf [indexBlockDescSize]byte
buf := make([]byte, indexBlockDescSize+len(d.extBitmap))
binary.BigEndian.PutUint64(buf[0:8], d.max)
binary.BigEndian.PutUint16(buf[8:10], d.entries)
binary.BigEndian.PutUint32(buf[10:14], d.id)
copy(buf[indexBlockDescSize:], d.extBitmap)
return buf[:]
}
// decode unpacks index block descriptor from byte stream.
// decode unpacks index block descriptor from byte stream. It's safe to mutate
// the provided byte stream after the function call.
func (d *indexBlockDesc) decode(blob []byte) {
d.max = binary.BigEndian.Uint64(blob[:8])
d.entries = binary.BigEndian.Uint16(blob[8:10])
d.id = binary.BigEndian.Uint32(blob[10:14])
d.extBitmap = bytes.Clone(blob[indexBlockDescSize:])
}
// copy returns a deep-copied object.
func (d *indexBlockDesc) copy() *indexBlockDesc {
return &indexBlockDesc{
max: d.max,
entries: d.entries,
id: d.id,
extBitmap: bytes.Clone(d.extBitmap),
}
}
// parseIndexBlock parses the index block with the supplied byte stream.
@ -97,20 +109,38 @@ func (d *indexBlockDesc) decode(blob []byte) {
// A uint16 can cover offsets in the range [0, 65536), which is more than enough
// to store 4096 integers.
//
// Each chunk begins with the full value of the first integer, followed by
// subsequent integers representing the differences between the current value
// and the preceding one. Integers are encoded with variable-size for best
// storage efficiency. Each chunk can be illustrated as below.
// Each chunk begins with a full integer value for the first element, followed
// by subsequent integers encoded as differences (deltas) from their preceding
// values. All integers use variable-length encoding for optimal space efficiency.
//
// Restart ---> +----------------+
// | Full integer |
// +----------------+
// | Diff with prev |
// +----------------+
// | ... |
// +----------------+
// | Diff with prev |
// +----------------+
// In the updated format, each element in the chunk may optionally include an
// "extension" section. If an extension is present, it starts with a var-size
// integer indicating the length of the remaining extension payload, followed by
// that many bytes. If no extension is present, the element format is identical
// to the original version (i.e., only the integer or delta value is encoded).
//
// In the trienode history index, the extension field contains the list of
// trie node IDs that fall within this range. For the given state transition,
// these IDs represent the specific nodes in this range that were mutated.
//
// Whether an element includes an extension is determined by the block reader
// based on the specification. Conceptually, a chunk is structured as:
//
// Restart ---> +----------------+
// | Full integer |
// +----------------+
// | (Extension?) |
// +----------------+
// | Diff with prev |
// +----------------+
// | (Extension?) |
// +----------------+
// | ... |
// +----------------+
// | Diff with prev |
// +----------------+
// | (Extension?) |
// +----------------+
//
// Empty index block is regarded as invalid.
func parseIndexBlock(blob []byte) ([]uint16, []byte, error) {
@ -148,24 +178,26 @@ func parseIndexBlock(blob []byte) ([]uint16, []byte, error) {
type blockReader struct {
restarts []uint16
data []byte
hasExt bool
}
// newBlockReader constructs the block reader with the supplied block data.
func newBlockReader(blob []byte) (*blockReader, error) {
func newBlockReader(blob []byte, hasExt bool) (*blockReader, error) {
restarts, data, err := parseIndexBlock(blob)
if err != nil {
return nil, err
}
return &blockReader{
restarts: restarts,
data: data, // safe to own the slice
data: data, // safe to own the slice
hasExt: hasExt, // flag whether extension should be resolved
}, nil
}
// readGreaterThan locates the first element in the block that is greater than
// the specified value. If no such element is found, MaxUint64 is returned.
func (br *blockReader) readGreaterThan(id uint64) (uint64, error) {
it := newBlockIterator(br.data, br.restarts)
it := br.newIterator(nil)
found := it.SeekGT(id)
if err := it.Error(); err != nil {
return 0, err
@ -180,17 +212,19 @@ type blockWriter struct {
desc *indexBlockDesc // Descriptor of the block
restarts []uint16 // Offsets into the data slice, marking the start of each section
data []byte // Aggregated encoded data slice
hasExt bool // Flag whether the extension field for each element exists
}
// newBlockWriter constructs a block writer. In addition to the existing data
// and block description, it takes an element ID and prunes all existing elements
// above that ID. It's essential as the recovery mechanism after unclean shutdown
// during the history indexing.
func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64) (*blockWriter, error) {
func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64, hasExt bool) (*blockWriter, error) {
if len(blob) == 0 {
return &blockWriter{
desc: desc,
data: make([]byte, 0, 1024),
desc: desc,
data: make([]byte, 0, 1024),
hasExt: hasExt,
}, nil
}
restarts, data, err := parseIndexBlock(blob)
@ -201,6 +235,7 @@ func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64) (*blockWrit
desc: desc,
restarts: restarts,
data: data, // safe to own the slice
hasExt: hasExt,
}
var trimmed int
for !writer.empty() && writer.last() > limit {
@ -215,9 +250,26 @@ func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64) (*blockWrit
return writer, nil
}
// setBitmap applies the given extension elements into the bitmap.
func (b *blockWriter) setBitmap(ext []uint16) {
for _, n := range ext {
// Node ID zero is intentionally filtered out. Any element in this range
// can indicate that the sub-tree's root node was mutated, so storing zero
// is redundant and saves one byte for bitmap.
if n != 0 {
setBit(b.desc.extBitmap, int(n-1))
}
}
}
// append adds a new element to the block. The new element must be greater than
// the previous one. The provided ID is assumed to always be greater than 0.
func (b *blockWriter) append(id uint64) error {
//
// ext refers to the optional extension field attached to the appended element.
// This extension mechanism is used by trie-node history and represents a list of
// trie node IDs that fall within the range covered by the index element
// (typically corresponding to a sub-trie in trie-node history).
func (b *blockWriter) append(id uint64, ext []uint16) error {
if id == 0 {
return errors.New("invalid zero id")
}
@ -244,13 +296,29 @@ func (b *blockWriter) append(id uint64) error {
// element.
b.data = binary.AppendUvarint(b.data, id-b.desc.max)
}
// Extension validation
if (len(ext) == 0) != !b.hasExt {
if len(ext) == 0 {
return errors.New("missing extension")
}
return errors.New("unexpected extension")
}
// Append the extension if it is not nil. The extension is prefixed with a
// length indicator, and the block reader MUST understand this scheme and
// decode the extension accordingly.
if len(ext) > 0 {
b.setBitmap(ext)
enc := encodeIDs(ext)
b.data = binary.AppendUvarint(b.data, uint64(len(enc)))
b.data = append(b.data, enc...)
}
b.desc.entries++
b.desc.max = id
return nil
}
// scanSection traverses the specified section and terminates if fn returns true.
func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) {
func (b *blockWriter) scanSection(section int, fn func(uint64, int, []uint16) bool) error {
var (
value uint64
start = int(b.restarts[section])
@ -269,28 +337,47 @@ func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) {
} else {
value += x
}
if fn(value, pos) {
return
// Resolve the extension if exists
var (
err error
ext []uint16
extLen int
)
if b.hasExt {
l, ln := binary.Uvarint(b.data[pos+n:])
extLen = ln + int(l)
ext, err = decodeIDs(b.data[pos+n+ln : pos+n+extLen])
}
if err != nil {
return err
}
if fn(value, pos, ext) {
return nil
}
// Shift to next position
pos += n
pos += extLen
}
return nil
}
// sectionLast returns the last element in the specified section.
func (b *blockWriter) sectionLast(section int) uint64 {
func (b *blockWriter) sectionLast(section int) (uint64, error) {
var n uint64
b.scanSection(section, func(v uint64, _ int) bool {
if err := b.scanSection(section, func(v uint64, _ int, _ []uint16) bool {
n = v
return false
})
return n
}); err != nil {
return 0, err
}
return n, nil
}
// sectionSearch looks up the specified value in the given section,
// the position and the preceding value will be returned if found.
// It assumes that the preceding element exists in the section.
func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int) {
b.scanSection(section, func(v uint64, p int) bool {
func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int, err error) {
if err := b.scanSection(section, func(v uint64, p int, _ []uint16) bool {
if n == v {
pos = p
found = true
@ -298,8 +385,24 @@ func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uin
}
prev = v
return false // continue iteration
})
return found, prev, pos
}); err != nil {
return false, 0, 0, err
}
return found, prev, pos, nil
}
// rebuildBitmap scans the entire block and rebuilds the bitmap.
func (b *blockWriter) rebuildBitmap() error {
clear(b.desc.extBitmap)
for i := 0; i < len(b.restarts); i++ {
if err := b.scanSection(i, func(v uint64, p int, ext []uint16) bool {
b.setBitmap(ext)
return false // continue iteration
}); err != nil {
return err
}
}
return nil
}
// pop removes the last element from the block. The assumption is held that block
@ -315,6 +418,7 @@ func (b *blockWriter) pop(id uint64) error {
if b.desc.entries == 1 {
b.desc.max = 0
b.desc.entries = 0
clear(b.desc.extBitmap)
b.restarts = nil
b.data = b.data[:0]
return nil
@ -324,28 +428,36 @@ func (b *blockWriter) pop(id uint64) error {
if b.desc.entries%indexBlockRestartLen == 1 {
b.data = b.data[:b.restarts[len(b.restarts)-1]]
b.restarts = b.restarts[:len(b.restarts)-1]
b.desc.max = b.sectionLast(len(b.restarts) - 1)
last, err := b.sectionLast(len(b.restarts) - 1)
if err != nil {
return err
}
b.desc.max = last
b.desc.entries -= 1
return nil
return b.rebuildBitmap()
}
// Look up the element preceding the one to be popped, in order to update
// the maximum element in the block.
found, prev, pos := b.sectionSearch(len(b.restarts)-1, id)
found, prev, pos, err := b.sectionSearch(len(b.restarts)-1, id)
if err != nil {
return err
}
if !found {
return fmt.Errorf("pop element is not found, last: %d, this: %d", b.desc.max, id)
}
b.desc.max = prev
b.data = b.data[:pos]
b.desc.entries -= 1
return nil
return b.rebuildBitmap()
}
func (b *blockWriter) empty() bool {
return b.desc.empty()
}
func (b *blockWriter) full() bool {
return b.desc.full()
func (b *blockWriter) estimateFull(ext []uint16) bool {
size := 8 + 2*len(ext)
return len(b.data)+size > indexBlockMaxSize
}
// last returns the last element in the block. It should only be called when

238
triedb/pathdb/history_index_block_test.go
View file

@ -17,6 +17,7 @@
package pathdb
import (
"bytes"
"math"
"math/rand"
"slices"
@ -24,16 +25,36 @@ import (
"testing"
)
func randomExt(bitmapSize int, n int) []uint16 {
if bitmapSize == 0 {
return nil
}
var (
limit = bitmapSize * 8
extList []uint16
)
for i := 0; i < n; i++ {
extList = append(extList, uint16(rand.Intn(limit+1)))
}
return extList
}
func TestBlockReaderBasic(t *testing.T) {
testBlockReaderBasic(t, 0)
testBlockReaderBasic(t, 2)
testBlockReaderBasic(t, 34)
}
func testBlockReaderBasic(t *testing.T, bitmapSize int) {
elements := []uint64{
1, 5, 10, 11, 20,
}
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
bw.append(elements[i], randomExt(bitmapSize, 5))
}
br, err := newBlockReader(bw.finish())
br, err := newBlockReader(bw.finish(), bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct the block reader, %v", err)
}
@ -60,18 +81,24 @@ func TestBlockReaderBasic(t *testing.T) {
}
func TestBlockReaderLarge(t *testing.T) {
testBlockReaderLarge(t, 0)
testBlockReaderLarge(t, 2)
testBlockReaderLarge(t, 34)
}
func testBlockReaderLarge(t *testing.T, bitmapSize int) {
var elements []uint64
for i := 0; i < 1000; i++ {
elements = append(elements, rand.Uint64())
}
slices.Sort(elements)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
bw.append(elements[i], randomExt(bitmapSize, 5))
}
br, err := newBlockReader(bw.finish())
br, err := newBlockReader(bw.finish(), bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct the block reader, %v", err)
}
@ -95,26 +122,32 @@ func TestBlockReaderLarge(t *testing.T) {
}
func TestBlockWriterBasic(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
testBlockWriteBasic(t, 0)
testBlockWriteBasic(t, 2)
testBlockWriteBasic(t, 34)
}
func testBlockWriteBasic(t *testing.T, bitmapSize int) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
if !bw.empty() {
t.Fatal("expected empty block")
}
bw.append(2)
if err := bw.append(1); err == nil {
bw.append(2, randomExt(bitmapSize, 5))
if err := bw.append(1, randomExt(bitmapSize, 5)); err == nil {
t.Fatal("out-of-order insertion is not expected")
}
var maxElem uint64
for i := 0; i < 10; i++ {
bw.append(uint64(i + 3))
bw.append(uint64(i+3), randomExt(bitmapSize, 5))
maxElem = uint64(i + 3)
}
bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0), maxElem)
bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0, bitmapSize), maxElem, bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct the block writer, %v", err)
}
for i := 0; i < 10; i++ {
if err := bw.append(uint64(i + 100)); err != nil {
if err := bw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil {
t.Fatalf("Failed to append value %d: %v", i, err)
}
}
@ -122,58 +155,38 @@ func TestBlockWriterBasic(t *testing.T) {
}
func TestBlockWriterWithLimit(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
testBlockWriterWithLimit(t, 0)
testBlockWriterWithLimit(t, 2)
testBlockWriterWithLimit(t, 34)
}
var maxElem uint64
for i := 0; i < indexBlockRestartLen*2; i++ {
bw.append(uint64(i + 1))
maxElem = uint64(i + 1)
}
func testBlockWriterWithLimit(t *testing.T, bitmapSize int) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
suites := []struct {
limit uint64
expMax uint64
}{
// nothing to truncate
{
maxElem, maxElem,
},
// truncate the last element
{
maxElem - 1, maxElem - 1,
},
// truncation around the restart boundary
{
uint64(indexBlockRestartLen + 1),
uint64(indexBlockRestartLen + 1),
},
// truncation around the restart boundary
{
uint64(indexBlockRestartLen),
uint64(indexBlockRestartLen),
},
{
uint64(1), uint64(1),
},
// truncate the entire block, it's in theory invalid
{
uint64(0), uint64(0),
},
var bitmaps [][]byte
for i := 0; i < indexBlockRestartLen+2; i++ {
bw.append(uint64(i+1), randomExt(bitmapSize, 5))
bitmaps = append(bitmaps, bytes.Clone(bw.desc.extBitmap))
}
for i, suite := range suites {
desc := *bw.desc
block, err := newBlockWriter(bw.finish(), &desc, suite.limit)
for i := 0; i < indexBlockRestartLen+2; i++ {
limit := uint64(i + 1)
desc := bw.desc.copy()
block, err := newBlockWriter(bytes.Clone(bw.finish()), desc, limit, bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct the block writer, %v", err)
}
if block.desc.max != suite.expMax {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, suite.expMax)
if block.desc.max != limit {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, limit)
}
if !bytes.Equal(desc.extBitmap, bitmaps[i]) {
t.Fatalf("Test %d, unexpected bitmap, got: %v, want: %v", i, block.desc.extBitmap, bitmaps[i])
}
// Re-fill the elements
var maxElem uint64
for elem := suite.limit + 1; elem < indexBlockRestartLen*4; elem++ {
if err := block.append(elem); err != nil {
for elem := limit + 1; elem < indexBlockRestartLen+4; elem++ {
if err := block.append(elem, randomExt(bitmapSize, 5)); err != nil {
t.Fatalf("Failed to append value %d: %v", elem, err)
}
maxElem = elem
@ -185,9 +198,15 @@ func TestBlockWriterWithLimit(t *testing.T) {
}
func TestBlockWriterDelete(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
testBlockWriterDelete(t, 0)
testBlockWriterDelete(t, 2)
testBlockWriterDelete(t, 34)
}
func testBlockWriterDelete(t *testing.T, bitmapSize int) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
for i := 0; i < 10; i++ {
bw.append(uint64(i + 1))
bw.append(uint64(i+1), randomExt(bitmapSize, 5))
}
// Pop unknown id, the request should be rejected
if err := bw.pop(100); err == nil {
@ -209,12 +228,18 @@ func TestBlockWriterDelete(t *testing.T) {
}
func TestBlcokWriterDeleteWithData(t *testing.T) {
testBlcokWriterDeleteWithData(t, 0)
testBlcokWriterDeleteWithData(t, 2)
testBlcokWriterDeleteWithData(t, 34)
}
func testBlcokWriterDeleteWithData(t *testing.T, bitmapSize int) {
elements := []uint64{
1, 5, 10, 11, 20,
}
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
bw.append(elements[i], randomExt(bitmapSize, 5))
}
// Re-construct the block writer with data
@ -223,7 +248,10 @@ func TestBlcokWriterDeleteWithData(t *testing.T) {
max: 20,
entries: 5,
}
bw, err := newBlockWriter(bw.finish(), desc, elements[len(elements)-1])
if bitmapSize > 0 {
desc.extBitmap = make([]byte, bitmapSize)
}
bw, err := newBlockWriter(bw.finish(), desc, elements[len(elements)-1], bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct block writer %v", err)
}
@ -234,7 +262,7 @@ func TestBlcokWriterDeleteWithData(t *testing.T) {
newTail := elements[i-1]
// Ensure the element can still be queried with no issue
br, err := newBlockReader(bw.finish())
br, err := newBlockReader(bw.finish(), bitmapSize != 0)
if err != nil {
t.Fatalf("Failed to construct the block reader, %v", err)
}
@ -266,29 +294,60 @@ func TestBlcokWriterDeleteWithData(t *testing.T) {
}
func TestCorruptedIndexBlock(t *testing.T) {
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false)
var maxElem uint64
for i := 0; i < 10; i++ {
bw.append(uint64(i + 1))
bw.append(uint64(i+1), nil)
maxElem = uint64(i + 1)
}
buf := bw.finish()
// Mutate the buffer manually
buf[len(buf)-1]++
_, err := newBlockWriter(buf, newIndexBlockDesc(0), maxElem)
_, err := newBlockWriter(buf, newIndexBlockDesc(0, 0), maxElem, false)
if err == nil {
t.Fatal("Corrupted index block data is not detected")
}
}
// BenchmarkParseIndexBlock benchmarks the performance of parseIndexBlock.
//
// goos: darwin
// goarch: arm64
// pkg: github.com/ethereum/go-ethereum/triedb/pathdb
// cpu: Apple M1 Pro
// BenchmarkParseIndexBlock
// BenchmarkParseIndexBlock-8 35829495 34.16 ns/op
func BenchmarkParseIndexBlock(b *testing.B) {
// Generate a realistic index block blob
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false)
for i := 0; i < 4096; i++ {
bw.append(uint64(i * 2))
bw.append(uint64(i*2), nil)
}
blob := bw.finish()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _, err := parseIndexBlock(blob)
if err != nil {
b.Fatalf("parseIndexBlock failed: %v", err)
}
}
}
// goos: darwin
// goarch: arm64
// pkg: github.com/ethereum/go-ethereum/triedb/pathdb
// cpu: Apple M1 Pro
// BenchmarkParseIndexBlockWithExt
// BenchmarkParseIndexBlockWithExt-8 35773242 33.72 ns/op
func BenchmarkParseIndexBlockWithExt(b *testing.B) {
// Generate a realistic index block blob
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 34), 0, true)
for i := 0; i < 4096; i++ {
id, ext := uint64(i*2), randomExt(34, 3)
bw.append(id, ext)
}
blob := bw.finish()
@ -302,21 +361,58 @@ func BenchmarkParseIndexBlock(b *testing.B) {
}
// BenchmarkBlockWriterAppend benchmarks the performance of indexblock.writer
//
// goos: darwin
// goarch: arm64
// pkg: github.com/ethereum/go-ethereum/triedb/pathdb
// cpu: Apple M1 Pro
// BenchmarkBlockWriterAppend
// BenchmarkBlockWriterAppend-8 293611083 4.113 ns/op 3 B/op 0 allocs/op
func BenchmarkBlockWriterAppend(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
var blockID uint32
desc := newIndexBlockDesc(blockID)
writer, _ := newBlockWriter(nil, desc, 0)
desc := newIndexBlockDesc(blockID, 0)
writer, _ := newBlockWriter(nil, desc, 0, false)
for i := 0; i < b.N; i++ {
if writer.full() {
if writer.estimateFull(nil) {
blockID += 1
desc = newIndexBlockDesc(blockID)
writer, _ = newBlockWriter(nil, desc, 0)
desc = newIndexBlockDesc(blockID, 0)
writer, _ = newBlockWriter(nil, desc, 0, false)
}
if err := writer.append(writer.desc.max + 1); err != nil {
if err := writer.append(writer.desc.max+1, nil); err != nil {
b.Error(err)
}
}
}
// goos: darwin
// goarch: arm64
// pkg: github.com/ethereum/go-ethereum/triedb/pathdb
// cpu: Apple M1 Pro
// BenchmarkBlockWriterAppendWithExt
// BenchmarkBlockWriterAppendWithExt-8 11123844 103.6 ns/op 42 B/op 2 allocs/op
func BenchmarkBlockWriterAppendWithExt(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
var (
bitmapSize = 34
blockID uint32
)
desc := newIndexBlockDesc(blockID, bitmapSize)
writer, _ := newBlockWriter(nil, desc, 0, true)
for i := 0; i < b.N; i++ {
ext := randomExt(bitmapSize, 3)
if writer.estimateFull(ext) {
blockID += 1
desc = newIndexBlockDesc(blockID, bitmapSize)
writer, _ = newBlockWriter(nil, desc, 0, true)
}
if err := writer.append(writer.desc.max+1, ext); err != nil {
b.Error(err)
}
}

322
triedb/pathdb/history_index_iterator.go
View file

@ -40,31 +40,133 @@ type HistoryIndexIterator interface {
Error() error
}
// extFilter provides utilities for filtering index entries based on their
// extension field.
//
// It supports two primary operations:
//
// - determine whether a given target node ID or any of its descendants
// appears explicitly in the extension list.
//
// - determine whether a given target node ID or any of its descendants
// is marked in the extension bitmap.
//
// Together, these checks allow callers to efficiently filter out the irrelevant
// index entries during the lookup.
type extFilter uint16
// exists takes the entire extension field in the index block and determines
// whether the target ID or its descendants appears. Note, any of descendant
// can implicitly mean the presence of ancestor.
func (f extFilter) exists(ext []byte) (bool, error) {
fn := uint16(f)
list, err := decodeIDs(ext)
if err != nil {
return false, err
}
for _, elem := range list {
if elem == fn {
return true, nil
}
if isAncestor(fn, elem) {
return true, nil
}
}
return false, nil
}
const (
// bitmapBytesTwoLevels is the size of the bitmap for two levels of the
// 16-ary tree (16 nodes total, excluding the root).
bitmapBytesTwoLevels = 2
// bitmapBytesThreeLevels is the size of the bitmap for three levels of
// the 16-ary tree (272 nodes total, excluding the root).
bitmapBytesThreeLevels = 34
// bitmapElementThresholdTwoLevels is the total number of elements in the
// two levels of a 16-ary tree (16 nodes total, excluding the root).
bitmapElementThresholdTwoLevels = 16
// bitmapElementThresholdThreeLevels is the total number of elements in the
// two levels of a 16-ary tree (16 nodes total, excluding the root).
bitmapElementThresholdThreeLevels = bitmapElementThresholdTwoLevels + 16*16
)
// contains takes the bitmap from the block metadata and determines whether the
// target ID or its descendants is marked in the bitmap. Note, any of descendant
// can implicitly mean the presence of ancestor.
func (f extFilter) contains(bitmap []byte) (bool, error) {
id := int(f)
if id == 0 {
return true, nil
}
n := id - 1 // apply the position shift for excluding root node
switch len(bitmap) {
case 0:
// Bitmap is not available, return "false positive"
return true, nil
case bitmapBytesTwoLevels:
// Bitmap for 2-level trie with at most 16 elements inside
if n >= bitmapElementThresholdTwoLevels {
return false, fmt.Errorf("invalid extension filter %d for 2 bytes bitmap", id)
}
return isBitSet(bitmap, n), nil
case bitmapBytesThreeLevels:
// Bitmap for 3-level trie with at most 16+16*16 elements inside
if n >= bitmapElementThresholdThreeLevels {
return false, fmt.Errorf("invalid extension filter %d for 34 bytes bitmap", id)
} else if n >= bitmapElementThresholdTwoLevels {
return isBitSet(bitmap, n), nil
} else {
// Check the element itself first
if isBitSet(bitmap, n) {
return true, nil
}
// Check descendants: the presence of any descendant implicitly
// represents a mutation of its ancestor.
return bitmap[2+2*n] != 0 || bitmap[3+2*n] != 0, nil
}
default:
return false, fmt.Errorf("unsupported bitmap size %d", len(bitmap))
}
}
// blockIterator is the iterator to traverse the indices within a single block.
type blockIterator struct {
// immutable fields
data []byte // Reference to the data segment within the block reader
restarts []uint16 // Offsets pointing to the restart sections within the data
hasExt bool // Flag whether the extension is included in the data
// Optional extension filter
filter *extFilter // Filters index entries based on the extension field.
// mutable fields
id uint64 // ID of the element at the iterators current position
ext []byte // Extension field of the element at the iterators current position
dataPtr int // Current read position within the data slice
restartPtr int // Index of the restart section where the iterator is currently positioned
exhausted bool // Flag whether the iterator has been exhausted
err error // Accumulated error during the traversal
}
func newBlockIterator(data []byte, restarts []uint16) *blockIterator {
func (br *blockReader) newIterator(filter *extFilter) *blockIterator {
it := &blockIterator{
data: data, // hold the slice directly with no deep copy
restarts: restarts, // hold the slice directly with no deep copy
data: br.data, // hold the slice directly with no deep copy
restarts: br.restarts, // hold the slice directly with no deep copy
hasExt: br.hasExt, // flag whether the extension should be resolved
filter: filter, // optional extension filter
}
it.reset()
return it
}
func (it *blockIterator) set(dataPtr int, restartPtr int, id uint64) {
func (it *blockIterator) set(dataPtr int, restartPtr int, id uint64, ext []byte) {
it.id = id
it.ext = ext
it.dataPtr = dataPtr
it.restartPtr = restartPtr
it.exhausted = dataPtr == len(it.data)
@ -79,6 +181,8 @@ func (it *blockIterator) setErr(err error) {
func (it *blockIterator) reset() {
it.id = 0
it.ext = nil
it.dataPtr = -1
it.restartPtr = -1
it.exhausted = false
@ -90,12 +194,26 @@ func (it *blockIterator) reset() {
}
}
// SeekGT moves the iterator to the first element whose id is greater than the
func (it *blockIterator) resolveExt(pos int) ([]byte, int, error) {
if !it.hasExt {
return nil, 0, nil
}
length, n := binary.Uvarint(it.data[pos:])
if n <= 0 {
return nil, 0, fmt.Errorf("too short for extension, pos: %d, datalen: %d", pos, len(it.data))
}
if len(it.data[pos+n:]) < int(length) {
return nil, 0, fmt.Errorf("too short for extension, pos: %d, length: %d, datalen: %d", pos, length, len(it.data))
}
return it.data[pos+n : pos+n+int(length)], n + int(length), nil
}
// seekGT moves the iterator to the first element whose id is greater than the
// given number. It returns whether such element exists.
//
// Note, this operation will unset the exhausted status and subsequent traversal
// is allowed.
func (it *blockIterator) SeekGT(id uint64) bool {
func (it *blockIterator) seekGT(id uint64) bool {
if it.err != nil {
return false
}
@ -112,11 +230,20 @@ func (it *blockIterator) SeekGT(id uint64) bool {
return false
}
if index == 0 {
item, n := binary.Uvarint(it.data[it.restarts[0]:])
pos := int(it.restarts[0])
item, n := binary.Uvarint(it.data[pos:])
if n <= 0 {
it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[0]))
return false
}
pos = pos + n
// If the restart size is 1, then the restart pointer shouldn't be 0.
// It's not practical and should be denied in the first place.
it.set(int(it.restarts[0])+n, 0, item)
ext, shift, err := it.resolveExt(pos)
if err != nil {
it.setErr(err)
return false
}
it.set(pos+shift, 0, item, ext)
return true
}
var (
@ -154,11 +281,18 @@ func (it *blockIterator) SeekGT(id uint64) bool {
}
pos += n
ext, shift, err := it.resolveExt(pos)
if err != nil {
it.setErr(err)
return false
}
pos += shift
if result > id {
if pos == limit {
it.set(pos, restartIndex+1, result)
it.set(pos, restartIndex+1, result, ext)
} else {
it.set(pos, restartIndex, result)
it.set(pos, restartIndex, result, ext)
}
return true
}
@ -170,8 +304,45 @@ func (it *blockIterator) SeekGT(id uint64) bool {
}
// The element which is the first one greater than the specified id
// is exactly the one located at the restart point.
item, n := binary.Uvarint(it.data[it.restarts[index]:])
it.set(int(it.restarts[index])+n, index, item)
pos = int(it.restarts[index])
item, n := binary.Uvarint(it.data[pos:])
if n <= 0 {
it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[index]))
return false
}
pos = pos + n
ext, shift, err := it.resolveExt(pos)
if err != nil {
it.setErr(err)
return false
}
it.set(pos+shift, index, item, ext)
return true
}
// SeekGT implements HistoryIndexIterator, is the wrapper of the seekGT with
// optional extension filter logic applied.
func (it *blockIterator) SeekGT(id uint64) bool {
if !it.seekGT(id) {
return false
}
if it.filter == nil {
return true
}
for {
found, err := it.filter.exists(it.ext)
if err != nil {
it.setErr(err)
return false
}
if found {
break
}
if !it.next() {
return false
}
}
return true
}
@ -183,10 +354,9 @@ func (it *blockIterator) init() {
it.restartPtr = 0
}
// Next implements the HistoryIndexIterator, moving the iterator to the next
// element. If the iterator has been exhausted, and boolean with false should
// be returned.
func (it *blockIterator) Next() bool {
// next moves the iterator to the next element. If the iterator has been exhausted,
// and boolean with false should be returned.
func (it *blockIterator) next() bool {
if it.exhausted || it.err != nil {
return false
}
@ -198,7 +368,6 @@ func (it *blockIterator) Next() bool {
it.setErr(fmt.Errorf("failed to decode item at pos %d", it.dataPtr))
return false
}
var val uint64
if it.dataPtr == int(it.restarts[it.restartPtr]) {
val = v
@ -206,16 +375,48 @@ func (it *blockIterator) Next() bool {
val = it.id + v
}
// Decode the extension field
ext, shift, err := it.resolveExt(it.dataPtr + n)
if err != nil {
it.setErr(err)
return false
}
// Move to the next restart section if the data pointer crosses the boundary
nextRestartPtr := it.restartPtr
if it.restartPtr < len(it.restarts)-1 && it.dataPtr+n == int(it.restarts[it.restartPtr+1]) {
if it.restartPtr < len(it.restarts)-1 && it.dataPtr+n+shift == int(it.restarts[it.restartPtr+1]) {
nextRestartPtr = it.restartPtr + 1
}
it.set(it.dataPtr+n, nextRestartPtr, val)
it.set(it.dataPtr+n+shift, nextRestartPtr, val, ext)
return true
}
// Next implements the HistoryIndexIterator, moving the iterator to the next
// element. It's a wrapper of next with optional extension filter logic applied.
func (it *blockIterator) Next() bool {
if !it.next() {
return false
}
if it.filter == nil {
return true
}
for {
found, err := it.filter.exists(it.ext)
if err != nil {
it.setErr(err)
return false
}
if found {
break
}
if !it.next() {
return false
}
}
return true
}
// ID implements HistoryIndexIterator, returning the id of the element where the
// iterator is positioned at.
func (it *blockIterator) ID() uint64 {
@ -226,15 +427,15 @@ func (it *blockIterator) ID() uint64 {
// Exhausting all the elements is not considered to be an error.
func (it *blockIterator) Error() error { return it.err }
// blockLoader defines the method to retrieve the specific block for reading.
type blockLoader func(id uint32) (*blockReader, error)
// indexIterator is an iterator to traverse the history indices belonging to the
// specific state entry.
type indexIterator struct {
// immutable fields
descList []*indexBlockDesc
loader blockLoader
reader *indexReader
// Optional extension filter
filter *extFilter
// mutable fields
blockIt *blockIterator
@ -243,10 +444,26 @@ type indexIterator struct {
err error
}
func newIndexIterator(descList []*indexBlockDesc, loader blockLoader) *indexIterator {
// newBlockIter initializes the block iterator with the specified block ID.
func (r *indexReader) newBlockIter(id uint32, filter *extFilter) (*blockIterator, error) {
br, ok := r.readers[id]
if !ok {
var err error
br, err = newBlockReader(readStateIndexBlock(r.state, r.db, id), r.bitmapSize != 0)
if err != nil {
return nil, err
}
r.readers[id] = br
}
return br.newIterator(filter), nil
}
// newIterator initializes the index iterator with the specified extension filter.
func (r *indexReader) newIterator(filter *extFilter) *indexIterator {
it := &indexIterator{
descList: descList,
loader: loader,
descList: r.descList,
reader: r,
filter: filter,
}
it.reset()
return it
@ -271,16 +488,32 @@ func (it *indexIterator) reset() {
}
func (it *indexIterator) open(blockPtr int) error {
id := it.descList[blockPtr].id
br, err := it.loader(id)
blockIt, err := it.reader.newBlockIter(it.descList[blockPtr].id, it.filter)
if err != nil {
return err
}
it.blockIt = newBlockIterator(br.data, br.restarts)
it.blockIt = blockIt
it.blockPtr = blockPtr
return nil
}
func (it *indexIterator) applyFilter(index int) (int, error) {
if it.filter == nil {
return index, nil
}
for index < len(it.descList) {
found, err := it.filter.contains(it.descList[index].extBitmap)
if err != nil {
return 0, err
}
if found {
break
}
index++
}
return index, nil
}
// SeekGT moves the iterator to the first element whose id is greater than the
// given number. It returns whether such element exists.
//
@ -293,6 +526,11 @@ func (it *indexIterator) SeekGT(id uint64) bool {
index := sort.Search(len(it.descList), func(i int) bool {
return id < it.descList[i].max
})
index, err := it.applyFilter(index)
if err != nil {
it.setErr(err)
return false
}
if index == len(it.descList) {
return false
}
@ -304,7 +542,13 @@ func (it *indexIterator) SeekGT(id uint64) bool {
return false
}
}
return it.blockIt.SeekGT(id)
// Terminate if the element which is greater than the id can be found in the
// last block; otherwise move to the next block. It may happen that all the
// target elements in this block are all less than id.
if it.blockIt.SeekGT(id) {
return true
}
return it.Next()
}
func (it *indexIterator) init() error {
@ -325,15 +569,23 @@ func (it *indexIterator) Next() bool {
it.setErr(err)
return false
}
if it.blockIt.Next() {
return true
}
if it.blockPtr == len(it.descList)-1 {
it.blockPtr++
index, err := it.applyFilter(it.blockPtr)
if err != nil {
it.setErr(err)
return false
}
it.blockPtr = index
if it.blockPtr == len(it.descList) {
it.exhausted = true
return false
}
if err := it.open(it.blockPtr + 1); err != nil {
if err := it.open(it.blockPtr); err != nil {
it.setErr(err)
return false
}

280
triedb/pathdb/history_index_iterator_test.go
View file

@ -19,7 +19,9 @@ package pathdb
import (
"errors"
"fmt"
"maps"
"math/rand"
"slices"
"sort"
"testing"
@ -28,12 +30,30 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
)
func makeTestIndexBlock(count int) ([]byte, []uint64) {
func checkExt(f *extFilter, ext []uint16) bool {
if f == nil {
return true
}
fn := uint16(*f)
for _, n := range ext {
if n == fn {
return true
}
if isAncestor(fn, n) {
return true
}
}
return false
}
func makeTestIndexBlock(count int, bitmapSize int) ([]byte, []uint64, [][]uint16) {
var (
marks = make(map[uint64]bool)
elements []uint64
elements = make([]uint64, 0, count)
extList = make([][]uint16, 0, count)
)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0), 0)
bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0)
for i := 0; i < count; i++ {
n := uint64(rand.Uint32())
if marks[n] {
@ -45,17 +65,20 @@ func makeTestIndexBlock(count int) ([]byte, []uint64) {
sort.Slice(elements, func(i, j int) bool { return elements[i] < elements[j] })
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
ext := randomExt(bitmapSize, 5)
extList = append(extList, ext)
bw.append(elements[i], ext)
}
data := bw.finish()
return data, elements
return data, elements, extList
}
func makeTestIndexBlocks(db ethdb.KeyValueStore, stateIdent stateIdent, count int) []uint64 {
func makeTestIndexBlocks(db ethdb.KeyValueStore, stateIdent stateIdent, count int, bitmapSize int) ([]uint64, [][]uint16) {
var (
marks = make(map[uint64]bool)
elements []uint64
extList [][]uint16
)
for i := 0; i < count; i++ {
n := uint64(rand.Uint32())
@ -67,15 +90,17 @@ func makeTestIndexBlocks(db ethdb.KeyValueStore, stateIdent stateIdent, count in
}
sort.Slice(elements, func(i, j int) bool { return elements[i] < elements[j] })
iw, _ := newIndexWriter(db, stateIdent, 0)
iw, _ := newIndexWriter(db, stateIdent, 0, bitmapSize)
for i := 0; i < len(elements); i++ {
iw.append(elements[i])
ext := randomExt(bitmapSize, 5)
extList = append(extList, ext)
iw.append(elements[i], ext)
}
batch := db.NewBatch()
iw.finish(batch)
batch.Write()
return elements
return elements, extList
}
func checkSeekGT(it HistoryIndexIterator, input uint64, exp bool, expVal uint64) error {
@ -113,43 +138,40 @@ func checkNext(it HistoryIndexIterator, values []uint64) error {
return it.Error()
}
func TestBlockIteratorSeekGT(t *testing.T) {
/* 0-size index block is not allowed
data, elements := makeTestIndexBlock(0)
testBlockIterator(t, data, elements)
*/
data, elements := makeTestIndexBlock(1)
testBlockIterator(t, data, elements)
data, elements = makeTestIndexBlock(indexBlockRestartLen)
testBlockIterator(t, data, elements)
data, elements = makeTestIndexBlock(3 * indexBlockRestartLen)
testBlockIterator(t, data, elements)
data, elements = makeTestIndexBlock(indexBlockEntriesCap)
testBlockIterator(t, data, elements)
}
func testBlockIterator(t *testing.T, data []byte, elements []uint64) {
br, err := newBlockReader(data)
if err != nil {
t.Fatalf("Failed to open the block for reading, %v", err)
func verifySeekGT(t *testing.T, elements []uint64, ext [][]uint16, newIter func(filter *extFilter) HistoryIndexIterator) {
set := make(map[extFilter]bool)
for _, extList := range ext {
for _, f := range extList {
set[extFilter(f)] = true
}
}
it := newBlockIterator(br.data, br.restarts)
filters := slices.Collect(maps.Keys(set))
for i := 0; i < 128; i++ {
var filter *extFilter
if rand.Intn(2) == 0 && len(filters) > 0 {
filter = &filters[rand.Intn(len(filters))]
} else {
filter = nil
}
var input uint64
if rand.Intn(2) == 0 {
input = elements[rand.Intn(len(elements))]
} else {
input = uint64(rand.Uint32())
}
index := sort.Search(len(elements), func(i int) bool {
return elements[i] > input
})
for index < len(elements) {
if checkExt(filter, ext[index]) {
break
}
index++
}
var (
exp bool
expVal uint64
@ -160,10 +182,17 @@ func testBlockIterator(t *testing.T, data []byte, elements []uint64) {
} else {
exp = true
expVal = elements[index]
if index < len(elements) {
remains = elements[index+1:]
index++
for index < len(elements) {
if checkExt(filter, ext[index]) {
remains = append(remains, elements[index])
}
index++
}
}
it := newIter(filter)
if err := checkSeekGT(it, input, exp, expVal); err != nil {
t.Fatal(err)
}
@ -175,62 +204,71 @@ func testBlockIterator(t *testing.T, data []byte, elements []uint64) {
}
}
func verifyTraversal(t *testing.T, elements []uint64, ext [][]uint16, newIter func(filter *extFilter) HistoryIndexIterator) {
set := make(map[extFilter]bool)
for _, extList := range ext {
for _, f := range extList {
set[extFilter(f)] = true
}
}
filters := slices.Collect(maps.Keys(set))
for i := 0; i < 16; i++ {
var filter *extFilter
if len(filters) > 0 {
filter = &filters[rand.Intn(len(filters))]
} else {
filter = nil
}
it := newIter(filter)
var (
pos int
exp []uint64
)
for pos < len(elements) {
if checkExt(filter, ext[pos]) {
exp = append(exp, elements[pos])
}
pos++
}
if err := checkNext(it, exp); err != nil {
t.Fatal(err)
}
}
}
func TestBlockIteratorSeekGT(t *testing.T) {
for _, size := range []int{0, 2, 34} {
for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} {
data, elements, ext := makeTestIndexBlock(n, size)
verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator {
br, err := newBlockReader(data, size != 0)
if err != nil {
t.Fatalf("Failed to open the block for reading, %v", err)
}
return br.newIterator(filter)
})
}
}
}
func TestIndexIteratorSeekGT(t *testing.T) {
ident := newAccountIdent(common.Hash{0x1})
dbA := rawdb.NewMemoryDatabase()
testIndexIterator(t, ident, dbA, makeTestIndexBlocks(dbA, ident, 1))
for _, size := range []int{0, 2, 34} {
for _, n := range []int{1, 4096, 3 * 4096} {
db := rawdb.NewMemoryDatabase()
elements, ext := makeTestIndexBlocks(db, ident, n, size)
dbB := rawdb.NewMemoryDatabase()
testIndexIterator(t, ident, dbB, makeTestIndexBlocks(dbB, ident, 3*indexBlockEntriesCap))
dbC := rawdb.NewMemoryDatabase()
testIndexIterator(t, ident, dbC, makeTestIndexBlocks(dbC, ident, indexBlockEntriesCap-1))
dbD := rawdb.NewMemoryDatabase()
testIndexIterator(t, ident, dbD, makeTestIndexBlocks(dbD, ident, indexBlockEntriesCap+1))
}
func testIndexIterator(t *testing.T, stateIdent stateIdent, db ethdb.Database, elements []uint64) {
ir, err := newIndexReader(db, stateIdent)
if err != nil {
t.Fatalf("Failed to open the index reader, %v", err)
}
it := newIndexIterator(ir.descList, func(id uint32) (*blockReader, error) {
return newBlockReader(readStateIndexBlock(stateIdent, db, id))
})
for i := 0; i < 128; i++ {
var input uint64
if rand.Intn(2) == 0 {
input = elements[rand.Intn(len(elements))]
} else {
input = uint64(rand.Uint32())
}
index := sort.Search(len(elements), func(i int) bool {
return elements[i] > input
})
var (
exp bool
expVal uint64
remains []uint64
)
if index == len(elements) {
exp = false
} else {
exp = true
expVal = elements[index]
if index < len(elements) {
remains = elements[index+1:]
}
}
if err := checkSeekGT(it, input, exp, expVal); err != nil {
t.Fatal(err)
}
if exp {
if err := checkNext(it, remains); err != nil {
t.Fatal(err)
}
verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator {
ir, err := newIndexReader(db, ident, size)
if err != nil {
t.Fatalf("Failed to open the index reader, %v", err)
}
return ir.newIterator(filter)
})
}
}
}
@ -242,56 +280,36 @@ func TestBlockIteratorTraversal(t *testing.T) {
testBlockIterator(t, data, elements)
*/
data, elements := makeTestIndexBlock(1)
testBlockIteratorTraversal(t, data, elements)
for _, size := range []int{0, 2, 34} {
for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} {
data, elements, ext := makeTestIndexBlock(n, size)
data, elements = makeTestIndexBlock(indexBlockRestartLen)
testBlockIteratorTraversal(t, data, elements)
data, elements = makeTestIndexBlock(3 * indexBlockRestartLen)
testBlockIteratorTraversal(t, data, elements)
data, elements = makeTestIndexBlock(indexBlockEntriesCap)
testBlockIteratorTraversal(t, data, elements)
}
func testBlockIteratorTraversal(t *testing.T, data []byte, elements []uint64) {
br, err := newBlockReader(data)
if err != nil {
t.Fatalf("Failed to open the block for reading, %v", err)
}
it := newBlockIterator(br.data, br.restarts)
if err := checkNext(it, elements); err != nil {
t.Fatal(err)
verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator {
br, err := newBlockReader(data, size != 0)
if err != nil {
t.Fatalf("Failed to open the block for reading, %v", err)
}
return br.newIterator(filter)
})
}
}
}
func TestIndexIteratorTraversal(t *testing.T) {
ident := newAccountIdent(common.Hash{0x1})
dbA := rawdb.NewMemoryDatabase()
testIndexIteratorTraversal(t, ident, dbA, makeTestIndexBlocks(dbA, ident, 1))
for _, size := range []int{0, 2, 34} {
for _, n := range []int{1, 4096, 3 * 4096} {
db := rawdb.NewMemoryDatabase()
elements, ext := makeTestIndexBlocks(db, ident, n, size)
dbB := rawdb.NewMemoryDatabase()
testIndexIteratorTraversal(t, ident, dbB, makeTestIndexBlocks(dbB, ident, 3*indexBlockEntriesCap))
dbC := rawdb.NewMemoryDatabase()
testIndexIteratorTraversal(t, ident, dbC, makeTestIndexBlocks(dbC, ident, indexBlockEntriesCap-1))
dbD := rawdb.NewMemoryDatabase()
testIndexIteratorTraversal(t, ident, dbD, makeTestIndexBlocks(dbD, ident, indexBlockEntriesCap+1))
}
func testIndexIteratorTraversal(t *testing.T, stateIdent stateIdent, db ethdb.KeyValueReader, elements []uint64) {
ir, err := newIndexReader(db, stateIdent)
if err != nil {
t.Fatalf("Failed to open the index reader, %v", err)
}
it := newIndexIterator(ir.descList, func(id uint32) (*blockReader, error) {
return newBlockReader(readStateIndexBlock(stateIdent, db, id))
})
if err := checkNext(it, elements); err != nil {
t.Fatal(err)
verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator {
ir, err := newIndexReader(db, ident, size)
if err != nil {
t.Fatalf("Failed to open the index reader, %v", err)
}
return ir.newIterator(filter)
})
}
}
}

192
triedb/pathdb/history_index_test.go
View file

@ -29,19 +29,25 @@ import (
)
func TestIndexReaderBasic(t *testing.T) {
testIndexReaderBasic(t, 0)
testIndexReaderBasic(t, 2)
testIndexReaderBasic(t, 34)
}
func testIndexReaderBasic(t *testing.T, bitmapSize int) {
elements := []uint64{
1, 5, 10, 11, 20,
}
db := rawdb.NewMemoryDatabase()
bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
bw.append(elements[i], randomExt(bitmapSize, 5))
}
batch := db.NewBatch()
bw.finish(batch)
batch.Write()
br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}))
br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize)
if err != nil {
t.Fatalf("Failed to construct the index reader, %v", err)
}
@ -68,22 +74,28 @@ func TestIndexReaderBasic(t *testing.T) {
}
func TestIndexReaderLarge(t *testing.T) {
testIndexReaderLarge(t, 0)
testIndexReaderLarge(t, 2)
testIndexReaderLarge(t, 34)
}
func testIndexReaderLarge(t *testing.T, bitmapSize int) {
var elements []uint64
for i := 0; i < 10*indexBlockEntriesCap; i++ {
for i := 0; i < 10*4096; i++ {
elements = append(elements, rand.Uint64())
}
slices.Sort(elements)
db := rawdb.NewMemoryDatabase()
bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
for i := 0; i < len(elements); i++ {
bw.append(elements[i])
bw.append(elements[i], randomExt(bitmapSize, 5))
}
batch := db.NewBatch()
bw.finish(batch)
batch.Write()
br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}))
br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize)
if err != nil {
t.Fatalf("Failed to construct the index reader, %v", err)
}
@ -107,7 +119,7 @@ func TestIndexReaderLarge(t *testing.T) {
}
func TestEmptyIndexReader(t *testing.T) {
br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa}))
br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa}), 0)
if err != nil {
t.Fatalf("Failed to construct the index reader, %v", err)
}
@ -121,27 +133,33 @@ func TestEmptyIndexReader(t *testing.T) {
}
func TestIndexWriterBasic(t *testing.T) {
testIndexWriterBasic(t, 0)
testIndexWriterBasic(t, 2)
testIndexWriterBasic(t, 34)
}
func testIndexWriterBasic(t *testing.T, bitmapSize int) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
iw.append(2)
if err := iw.append(1); err == nil {
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
iw.append(2, randomExt(bitmapSize, 5))
if err := iw.append(1, randomExt(bitmapSize, 5)); err == nil {
t.Fatal("out-of-order insertion is not expected")
}
var maxElem uint64
for i := 0; i < 10; i++ {
iw.append(uint64(i + 3))
iw.append(uint64(i+3), randomExt(bitmapSize, 5))
maxElem = uint64(i + 3)
}
batch := db.NewBatch()
iw.finish(batch)
batch.Write()
iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), maxElem)
iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize)
if err != nil {
t.Fatalf("Failed to construct the block writer, %v", err)
}
for i := 0; i < 10; i++ {
if err := iw.append(uint64(i + 100)); err != nil {
if err := iw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil {
t.Fatalf("Failed to append item, %v", err)
}
}
@ -149,61 +167,37 @@ func TestIndexWriterBasic(t *testing.T) {
}
func TestIndexWriterWithLimit(t *testing.T) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
testIndexWriterWithLimit(t, 0)
testIndexWriterWithLimit(t, 2)
testIndexWriterWithLimit(t, 34)
}
var maxElem uint64
for i := 0; i < indexBlockEntriesCap*2; i++ {
iw.append(uint64(i + 1))
maxElem = uint64(i + 1)
func testIndexWriterWithLimit(t *testing.T, bitmapSize int) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
// 200 iterations (with around 50 bytes extension) is enough to cross
// the block boundary (4096 bytes)
for i := 0; i < 200; i++ {
iw.append(uint64(i+1), randomExt(bitmapSize, 50))
}
batch := db.NewBatch()
iw.finish(batch)
batch.Write()
suites := []struct {
limit uint64
expMax uint64
}{
// nothing to truncate
{
maxElem, maxElem,
},
// truncate the last element
{
maxElem - 1, maxElem - 1,
},
// truncation around the block boundary
{
uint64(indexBlockEntriesCap + 1),
uint64(indexBlockEntriesCap + 1),
},
// truncation around the block boundary
{
uint64(indexBlockEntriesCap),
uint64(indexBlockEntriesCap),
},
{
uint64(1), uint64(1),
},
// truncate the entire index, it's in theory invalid
{
uint64(0), uint64(0),
},
}
for i, suite := range suites {
iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), suite.limit)
for i := 0; i < 200; i++ {
limit := uint64(i + 1)
iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize)
if err != nil {
t.Fatalf("Failed to construct the index writer, %v", err)
}
if iw.lastID != suite.expMax {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, suite.expMax)
if iw.lastID != limit {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit)
}
// Re-fill the elements
var maxElem uint64
for elem := suite.limit + 1; elem < indexBlockEntriesCap*4; elem++ {
if err := iw.append(elem); err != nil {
for elem := limit + 1; elem < 500; elem++ {
if err := iw.append(elem, randomExt(bitmapSize, 5)); err != nil {
t.Fatalf("Failed to append value %d: %v", elem, err)
}
maxElem = elem
@ -215,12 +209,20 @@ func TestIndexWriterWithLimit(t *testing.T) {
}
func TestIndexDeleterBasic(t *testing.T) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
testIndexDeleterBasic(t, 0)
testIndexDeleterBasic(t, 2)
testIndexDeleterBasic(t, 34)
}
func testIndexDeleterBasic(t *testing.T, bitmapSize int) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
// 200 iterations (with around 50 bytes extension) is enough to cross
// the block boundary (4096 bytes)
var maxElem uint64
for i := 0; i < indexBlockEntriesCap*4; i++ {
iw.append(uint64(i + 1))
for i := 0; i < 200; i++ {
iw.append(uint64(i+1), randomExt(bitmapSize, 50))
maxElem = uint64(i + 1)
}
batch := db.NewBatch()
@ -228,11 +230,11 @@ func TestIndexDeleterBasic(t *testing.T) {
batch.Write()
// Delete unknown id, the request should be rejected
id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), maxElem)
if err := id.pop(indexBlockEntriesCap * 5); err == nil {
id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize)
if err := id.pop(500); err == nil {
t.Fatal("Expect error to occur for unknown id")
}
for i := indexBlockEntriesCap * 4; i >= 1; i-- {
for i := 200; i >= 1; i-- {
if err := id.pop(uint64(i)); err != nil {
t.Fatalf("Unexpected error for element popping, %v", err)
}
@ -243,57 +245,33 @@ func TestIndexDeleterBasic(t *testing.T) {
}
func TestIndexDeleterWithLimit(t *testing.T) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0)
testIndexDeleterWithLimit(t, 0)
testIndexDeleterWithLimit(t, 2)
testIndexDeleterWithLimit(t, 34)
}
var maxElem uint64
for i := 0; i < indexBlockEntriesCap*2; i++ {
iw.append(uint64(i + 1))
maxElem = uint64(i + 1)
func testIndexDeleterWithLimit(t *testing.T, bitmapSize int) {
db := rawdb.NewMemoryDatabase()
iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize)
// 200 iterations (with around 50 bytes extension) is enough to cross
// the block boundary (4096 bytes)
for i := 0; i < 200; i++ {
iw.append(uint64(i+1), randomExt(bitmapSize, 50))
}
batch := db.NewBatch()
iw.finish(batch)
batch.Write()
suites := []struct {
limit uint64
expMax uint64
}{
// nothing to truncate
{
maxElem, maxElem,
},
// truncate the last element
{
maxElem - 1, maxElem - 1,
},
// truncation around the block boundary
{
uint64(indexBlockEntriesCap + 1),
uint64(indexBlockEntriesCap + 1),
},
// truncation around the block boundary
{
uint64(indexBlockEntriesCap),
uint64(indexBlockEntriesCap),
},
{
uint64(1), uint64(1),
},
// truncate the entire index, it's in theory invalid
{
uint64(0), uint64(0),
},
}
for i, suite := range suites {
id, err := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), suite.limit)
for i := 0; i < 200; i++ {
limit := uint64(i + 1)
id, err := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize)
if err != nil {
t.Fatalf("Failed to construct the index writer, %v", err)
}
if id.lastID != suite.expMax {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, id.lastID, suite.expMax)
if id.lastID != limit {
t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit)
}
// Keep removing elements
for elem := id.lastID; elem > 0; elem-- {
if err := id.pop(elem); err != nil {
@ -339,7 +317,7 @@ func TestBatchIndexerWrite(t *testing.T) {
}
}
for addrHash, indexes := range accounts {
ir, _ := newIndexReader(db, newAccountIdent(addrHash))
ir, _ := newIndexReader(db, newAccountIdent(addrHash), 0)
for i := 0; i < len(indexes)-1; i++ {
n, err := ir.readGreaterThan(indexes[i])
if err != nil {
@ -359,7 +337,7 @@ func TestBatchIndexerWrite(t *testing.T) {
}
for addrHash, slots := range storages {
for slotHash, indexes := range slots {
ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash))
ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash), 0)
for i := 0; i < len(indexes)-1; i++ {
n, err := ir.readGreaterThan(indexes[i])
if err != nil {

37
triedb/pathdb/history_indexer.go
View file

@ -29,12 +29,14 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"golang.org/x/exp/maps"
"golang.org/x/sync/errgroup"
)
const (
// The batch size for reading state histories
historyReadBatch = 1000
historyReadBatch = 1000
historyIndexBatch = 8 * 1024 * 1024 // The number of state history indexes for constructing or deleting as batch
stateHistoryIndexV0 = uint8(0) // initial version of state index structure
stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version
@ -120,18 +122,20 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) {
// batchIndexer is responsible for performing batch indexing or unindexing
// of historical data (e.g., state or trie node changes) atomically.
type batchIndexer struct {
index map[stateIdent][]uint64 // List of history IDs for tracked state entry
pending int // Number of entries processed in the current batch.
delete bool // Operation mode: true for unindex, false for index.
lastID uint64 // ID of the most recently processed history.
typ historyType // Type of history being processed (e.g., state or trienode).
db ethdb.KeyValueStore // Key-value database used to store or delete index data.
index map[stateIdent][]uint64 // List of history IDs for tracked state entry
ext map[stateIdent][][]uint16 // List of extension for each state element
pending int // Number of entries processed in the current batch.
delete bool // Operation mode: true for unindex, false for index.
lastID uint64 // ID of the most recently processed history.
typ historyType // Type of history being processed (e.g., state or trienode).
db ethdb.KeyValueStore // Key-value database used to store or delete index data.
}
// newBatchIndexer constructs the batch indexer with the supplied mode.
func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batchIndexer {
return &batchIndexer{
index: make(map[stateIdent][]uint64),
ext: make(map[stateIdent][][]uint16),
delete: delete,
typ: typ,
db: db,
@ -141,8 +145,10 @@ func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batc
// process traverses the state entries within the provided history and tracks the mutation
// records for them.
func (b *batchIndexer) process(h history, id uint64) error {
for ident := range h.forEach() {
b.index[ident] = append(b.index[ident], id)
for elem := range h.forEach() {
key := elem.key()
b.index[key] = append(b.index[key], id)
b.ext[key] = append(b.ext[key], elem.ext())
b.pending++
}
b.lastID = id
@ -189,14 +195,15 @@ func (b *batchIndexer) finish(force bool) error {
indexed = metadata.Last
}
for ident, list := range b.index {
ext := b.ext[ident]
eg.Go(func() error {
if !b.delete {
iw, err := newIndexWriter(b.db, ident, indexed)
iw, err := newIndexWriter(b.db, ident, indexed, ident.bloomSize())
if err != nil {
return err
}
for _, n := range list {
if err := iw.append(n); err != nil {
for i, n := range list {
if err := iw.append(n, ext[i]); err != nil {
return err
}
}
@ -204,7 +211,7 @@ func (b *batchIndexer) finish(force bool) error {
iw.finish(batch)
})
} else {
id, err := newIndexDeleter(b.db, ident, indexed)
id, err := newIndexDeleter(b.db, ident, indexed, ident.bloomSize())
if err != nil {
return err
}
@ -238,8 +245,10 @@ func (b *batchIndexer) finish(force bool) error {
return err
}
log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "size", common.StorageSize(batchSize), "elapsed", common.PrettyDuration(time.Since(start)))
b.pending = 0
b.index = make(map[stateIdent][]uint64)
maps.Clear(b.index)
maps.Clear(b.ext)
return nil
}

Some files were not shown because too many files have changed in this diff Show more