forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-25 06:06:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #24 from berachain/update-main

Browse source
This commit is contained in:
Cal Bera 2025-06-30 11:59:24 -07:00 committed by GitHub
commit debbcbb73f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 413 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/release.yml vendored
View file

@ -35,7 +35,7 @@ jobs:
- name: Build (amd64)
run: |
go run build/ci.go install -arch amd64 -dlgo
go run build/ci.go install -static -arch amd64 -dlgo
- name: Create archive (amd64)
run: |
@ -70,7 +70,7 @@ jobs:
- name: Build (arm64)
run: |
go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
go run build/ci.go install -static -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- name: Create archive (arm64)
run: |

8
cmd/geth/chaincmd.go
View file

@ -58,7 +58,7 @@ var (
ArgsUsage: "<genesisPath>",
Flags: slices.Concat([]cli.Flag{
utils.CachePreimagesFlag,
utils.OverridePrague,
utils.OverrideOsaka,
utils.OverrideVerkle,
}, utils.DatabaseFlags),
Description: `
@ -269,9 +269,9 @@ func initGenesis(ctx *cli.Context) error {
defer stack.Close()
var overrides core.ChainOverrides
if ctx.IsSet(utils.OverridePrague.Name) {
v := ctx.Uint64(utils.OverridePrague.Name)
overrides.OverridePrague = &v
if ctx.IsSet(utils.OverrideOsaka.Name) {
v := ctx.Uint64(utils.OverrideOsaka.Name)
overrides.OverrideOsaka = &v
}
if ctx.IsSet(utils.OverrideVerkle.Name) {
v := ctx.Uint64(utils.OverrideVerkle.Name)

6
cmd/geth/config.go
View file

@ -223,9 +223,9 @@ func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string {
// makeFullNode loads geth configuration and creates the Ethereum backend.
func makeFullNode(ctx *cli.Context) *node.Node {
stack, cfg := makeConfigNode(ctx)
if ctx.IsSet(utils.OverridePrague.Name) {
v := ctx.Uint64(utils.OverridePrague.Name)
cfg.Eth.OverridePrague = &v
if ctx.IsSet(utils.OverrideOsaka.Name) {
v := ctx.Uint64(utils.OverrideOsaka.Name)
cfg.Eth.OverrideOsaka = &v
}
if ctx.IsSet(utils.OverrideVerkle.Name) {
v := ctx.Uint64(utils.OverrideVerkle.Name)

2
cmd/geth/main.go
View file

@ -62,7 +62,7 @@ var (
utils.NoUSBFlag, // deprecated
utils.USBFlag,
utils.SmartCardDaemonPathFlag,
utils.OverridePrague,
utils.OverrideOsaka,
utils.OverrideVerkle,
utils.EnablePersonal, // deprecated
utils.TxPoolLocalsFlag,

6
cmd/utils/flags.go
View file

@ -254,9 +254,9 @@ var (
Value: 2048,
Category: flags.EthCategory,
}
OverridePrague = &cli.Uint64Flag{
Name: "override.prague",
Usage: "Manually specify the Prague fork timestamp, overriding the bundled setting",
OverrideOsaka = &cli.Uint64Flag{
Name: "override.osaka",
Usage: "Manually specify the Osaka fork timestamp, overriding the bundled setting",
Category: flags.EthCategory,
}
OverrideVerkle = &cli.Uint64Flag{

12
core/blockchain_reader.go
View file

@ -370,6 +370,13 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.statedb)
}
// HistoricState returns a historic state specified by the given root.
// Live states are not available and won't be served, please use `State`
// or `StateAt` instead.
func (bc *BlockChain) HistoricState(root common.Hash) (*state.StateDB, error) {
return state.New(root, state.NewHistoricDatabase(bc.db, bc.triedb))
}
// Config retrieves the chain's fork configuration.
func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig }
@ -419,6 +426,11 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) {
return bc.txIndexer.txIndexProgress(), nil
}
// StateIndexProgress returns the historical state indexing progress.
func (bc *BlockChain) StateIndexProgress() (uint64, error) {
return bc.triedb.IndexProgress()
}
// HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) {

6
core/genesis.go
View file

@ -262,7 +262,7 @@ func (e *GenesisMismatchError) Error() string {
// ChainOverrides contains the changes to chain config.
type ChainOverrides struct {
OverridePrague *uint64
OverrideOsaka *uint64
OverrideVerkle *uint64
}
@ -271,8 +271,8 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error {
if o == nil || cfg == nil {
return nil
}
if o.OverridePrague != nil {
cfg.PragueTime = o.OverridePrague
if o.OverrideOsaka != nil {
cfg.OsakaTime = o.OverrideOsaka
}
if o.OverrideVerkle != nil {
cfg.VerkleTime = o.OverrideVerkle

155
core/state/database_history.go Normal file
View file

@ -0,0 +1,155 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package state
import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/utils"
"github.com/ethereum/go-ethereum/triedb"
"github.com/ethereum/go-ethereum/triedb/pathdb"
)
// historicReader wraps a historical state reader defined in path database,
// providing historic state serving over the path scheme.
//
// TODO(rjl493456442): historicReader is not thread-safe and does not fully
// comply with the StateReader interface requirements, needs to be fixed.
// Currently, it is only used in a non-concurrent context, so it is safe for now.
type historicReader struct {
reader *pathdb.HistoricalStateReader
}
// newHistoricReader constructs a reader for historic state serving.
func newHistoricReader(r *pathdb.HistoricalStateReader) *historicReader {
return &historicReader{reader: r}
}
// Account implements StateReader, retrieving the account specified by the address.
//
// An error will be returned if the associated snapshot is already stale or
// the requested account is not yet covered by the snapshot.
//
// The returned account might be nil if it's not existent.
func (r *historicReader) Account(addr common.Address) (*types.StateAccount, error) {
account, err := r.reader.Account(addr)
if err != nil {
return nil, err
}
if account == nil {
return nil, nil
}
acct := &types.StateAccount{
Nonce: account.Nonce,
Balance: account.Balance,
CodeHash: account.CodeHash,
Root: common.BytesToHash(account.Root),
}
if len(acct.CodeHash) == 0 {
acct.CodeHash = types.EmptyCodeHash.Bytes()
}
if acct.Root == (common.Hash{}) {
acct.Root = types.EmptyRootHash
}
return acct, nil
}
// Storage implements StateReader, retrieving the storage slot specified by the
// address and slot key.
//
// An error will be returned if the associated snapshot is already stale or
// the requested storage slot is not yet covered by the snapshot.
//
// The returned storage slot might be empty if it's not existent.
func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) {
blob, err := r.reader.Storage(addr, key)
if err != nil {
return common.Hash{}, err
}
if len(blob) == 0 {
return common.Hash{}, nil
}
_, content, _, err := rlp.Split(blob)
if err != nil {
return common.Hash{}, err
}
var slot common.Hash
slot.SetBytes(content)
return slot, nil
}
// HistoricDB is the implementation of Database interface, with the ability to
// access historical state.
type HistoricDB struct {
disk ethdb.KeyValueStore
triedb *triedb.Database
codeCache *lru.SizeConstrainedCache[common.Hash, []byte]
codeSizeCache *lru.Cache[common.Hash, int]
pointCache *utils.PointCache
}
// NewHistoricDatabase creates a historic state database.
func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *HistoricDB {
return &HistoricDB{
disk: disk,
triedb: triedb,
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
pointCache: utils.NewPointCache(pointCacheSize),
}
}
// Reader implements Database interface, returning a reader of the specific state.
func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) {
hr, err := db.triedb.HistoricReader(stateRoot)
if err != nil {
return nil, err
}
return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), newHistoricReader(hr)), nil
}
// OpenTrie opens the main account trie. It's not supported by historic database.
func (db *HistoricDB) OpenTrie(root common.Hash) (Trie, error) {
return nil, errors.New("not implemented")
}
// OpenStorageTrie opens the storage trie of an account. It's not supported by
// historic database.
func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) {
return nil, errors.New("not implemented")
}
// PointCache returns the cache holding points used in verkle tree key computation
func (db *HistoricDB) PointCache() *utils.PointCache {
return db.pointCache
}
// TrieDB returns the underlying trie database for managing trie nodes.
func (db *HistoricDB) TrieDB() *triedb.Database {
return db.triedb
}
// Snapshot returns the underlying state snapshot.
func (db *HistoricDB) Snapshot() *snapshot.Tree {
return nil
}

10
eth/api_backend.go
View file

@ -237,9 +237,12 @@ func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.B
return nil, nil, errors.New("header not found")
}
stateDb, err := b.eth.BlockChain().StateAt(header.Root)
if err != nil {
stateDb, err = b.eth.BlockChain().HistoricState(header.Root)
if err != nil {
return nil, nil, err
}
}
return stateDb, header, nil
}
@ -259,9 +262,12 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN
return nil, nil, errors.New("hash is not currently canonical")
}
stateDb, err := b.eth.BlockChain().StateAt(header.Root)
if err != nil {
stateDb, err = b.eth.BlockChain().HistoricState(header.Root)
if err != nil {
return nil, nil, err
}
}
return stateDb, header, nil
}
return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
@ -397,6 +403,10 @@ func (b *EthAPIBackend) SyncProgress(ctx context.Context) ethereum.SyncProgress
prog.TxIndexFinishedBlocks = txProg.Indexed
prog.TxIndexRemainingBlocks = txProg.Remaining
}
remain, err := b.eth.blockchain.StateIndexProgress()
if err == nil {
prog.StateIndexRemaining = remain
}
return prog
}

15
eth/backend.go
View file

@ -140,7 +140,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice)
config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice)
}
if config.NoPruning && config.TrieDirtyCache > 0 {
if config.NoPruning && config.TrieDirtyCache > 0 && config.StateScheme == rawdb.HashScheme {
if config.SnapshotCache > 0 {
config.TrieCleanCache += config.TrieDirtyCache * 3 / 5
config.SnapshotCache += config.TrieDirtyCache * 2 / 5
@ -221,9 +221,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
}
var (
vmConfig = vm.Config{
EnablePreimageRecording: config.EnablePreimageRecording,
}
options = &core.BlockChainConfig{
TrieCleanLimit: config.TrieCleanCache,
NoPrefetch: config.NoPrefetch,
@ -236,7 +233,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
StateScheme: scheme,
ChainHistoryMode: config.HistoryMode,
TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)),
VmConfig: vmConfig,
VmConfig: vm.Config{
EnablePreimageRecording: config.EnablePreimageRecording,
},
}
)
@ -249,12 +248,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if err != nil {
return nil, fmt.Errorf("failed to create tracer %s: %v", config.VMTrace, err)
}
vmConfig.Tracer = t
options.VmConfig.Tracer = t
}
// Override the chain config with provided settings.
var overrides core.ChainOverrides
if config.OverridePrague != nil {
overrides.OverridePrague = config.OverridePrague
if config.OverrideOsaka != nil {
overrides.OverrideOsaka = config.OverrideOsaka
}
if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle

4
eth/downloader/api.go
View file

@ -81,6 +81,10 @@ func (api *DownloaderAPI) eventLoop() {
prog.TxIndexFinishedBlocks = txProg.Indexed
prog.TxIndexRemainingBlocks = txProg.Remaining
}
remain, err := api.chain.StateIndexProgress()
if err == nil {
prog.StateIndexRemaining = remain
}
return prog
}
)

4
eth/ethconfig/config.go
View file

@ -158,8 +158,8 @@ type Config struct {
// send-transaction variants. The unit is ether.
RPCTxFeeCap float64
// OverridePrague (TODO: remove after the fork)
OverridePrague *uint64 `toml:",omitempty"`
// OverrideOsaka (TODO: remove after the fork)
OverrideOsaka *uint64 `toml:",omitempty"`
// OverrideVerkle (TODO: remove after the fork)
OverrideVerkle *uint64 `toml:",omitempty"`

10
eth/ethconfig/gen_config.go
View file

@ -54,7 +54,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
RPCGasCap uint64
RPCEVMTimeout time.Duration
RPCTxFeeCap float64
OverridePrague *uint64 `toml:",omitempty"`
OverrideOsaka *uint64 `toml:",omitempty"`
OverrideVerkle *uint64 `toml:",omitempty"`
}
var enc Config
@ -95,7 +95,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.RPCGasCap = c.RPCGasCap
enc.RPCEVMTimeout = c.RPCEVMTimeout
enc.RPCTxFeeCap = c.RPCTxFeeCap
enc.OverridePrague = c.OverridePrague
enc.OverrideOsaka = c.OverrideOsaka
enc.OverrideVerkle = c.OverrideVerkle
return &enc, nil
}
@ -140,7 +140,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
RPCGasCap *uint64
RPCEVMTimeout *time.Duration
RPCTxFeeCap *float64
OverridePrague *uint64 `toml:",omitempty"`
OverrideOsaka *uint64 `toml:",omitempty"`
OverrideVerkle *uint64 `toml:",omitempty"`
}
var dec Config
@ -258,8 +258,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.RPCTxFeeCap != nil {
c.RPCTxFeeCap = *dec.RPCTxFeeCap
}
if dec.OverridePrague != nil {
c.OverridePrague = dec.OverridePrague
if dec.OverrideOsaka != nil {
c.OverrideOsaka = dec.OverrideOsaka
}
if dec.OverrideVerkle != nil {
c.OverrideVerkle = dec.OverrideVerkle

9
eth/state_accessor.go
View file

@ -182,10 +182,11 @@ func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), erro
if err == nil {
return statedb, noopReleaser, nil
}
// TODO historic state is not supported in path-based scheme.
// Fully archive node in pbss will be implemented by relying
// on state history, but needs more work on top.
return nil, nil, errors.New("historical state not available in path scheme yet")
statedb, err = eth.blockchain.HistoricState(block.Root())
if err == nil {
return statedb, noopReleaser, nil
}
return nil, nil, errors.New("historical state is not available")
}
// stateAtBlock retrieves the state database associated with a certain block.

2
ethclient/ethclient.go
View file

@ -789,6 +789,7 @@ type rpcProgress struct {
HealingBytecode hexutil.Uint64
TxIndexFinishedBlocks hexutil.Uint64
TxIndexRemainingBlocks hexutil.Uint64
StateIndexRemaining hexutil.Uint64
}
func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress {
@ -815,5 +816,6 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress {
HealingBytecode: uint64(p.HealingBytecode),
TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks),
TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks),
StateIndexRemaining: uint64(p.StateIndexRemaining),
}
}

3
graphql/graphql.go
View file

@ -1510,6 +1510,9 @@ func (s *SyncState) TxIndexFinishedBlocks() hexutil.Uint64 {
func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 {
return hexutil.Uint64(s.progress.TxIndexRemainingBlocks)
}
func (s *SyncState) StateIndexRemaining() hexutil.Uint64 {
return hexutil.Uint64(s.progress.StateIndexRemaining)
}
// Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not
// yet received the latest block headers from its peers. In case it is synchronizing:

5
interfaces.go
View file

@ -124,6 +124,9 @@ type SyncProgress struct {
// "transaction indexing" fields
TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed
TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet
// "historical state indexing" fields
StateIndexRemaining uint64 // Number of states remain unindexed
}
// Done returns the indicator if the initial sync is finished or not.
@ -131,7 +134,7 @@ func (prog SyncProgress) Done() bool {
if prog.CurrentBlock < prog.HighestBlock {
return false
}
return prog.TxIndexRemainingBlocks == 0
return prog.TxIndexRemainingBlocks == 0 && prog.StateIndexRemaining == 0
}
// ChainSyncReader wraps access to the node's current sync status. If there's no

1
internal/ethapi/api.go
View file

@ -170,6 +170,7 @@ func (api *EthereumAPI) Syncing(ctx context.Context) (interface{}, error) {
"healingBytecode": hexutil.Uint64(progress.HealingBytecode),
"txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks),
"txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks),
"stateIndexRemaining": hexutil.Uint64(progress.StateIndexRemaining),
}, nil
}

1
internal/jsre/deps/web3.js
View file

@ -3977,6 +3977,7 @@ var outputSyncingFormatter = function(result) {
result.healingBytecode = utils.toDecimal(result.healingBytecode);
result.txIndexFinishedBlocks = utils.toDecimal(result.txIndexFinishedBlocks);
result.txIndexRemainingBlocks = utils.toDecimal(result.txIndexRemainingBlocks);
result.stateIndexRemaining = utils.toDecimal(result.stateIndexRemaining)
return result;
};

4
node/database.go
View file

@ -109,7 +109,7 @@ func newLevelDBDatabase(file string, cache int, handles int, namespace string, r
return nil, err
}
log.Info("Using LevelDB as the backing database")
return rawdb.NewDatabase(db), nil
return db, nil
}
// newPebbleDBDatabase creates a persistent key-value database without a freezer
@ -119,5 +119,5 @@ func newPebbleDBDatabase(file string, cache int, handles int, namespace string,
if err != nil {
return nil, err
}
return rawdb.NewDatabase(db), nil
return db, nil
}

19
triedb/database.go
View file

@ -129,6 +129,15 @@ func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, er
return db.backend.StateReader(blockRoot)
}
// HistoricReader constructs a reader for accessing the requested historic state.
func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateReader, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return nil, errors.New("not supported")
}
return pdb.HistoricReader(root)
}
// Update performs a state transition by committing dirty nodes contained in the
// given set in order to update state from the specified parent to the specified
// root. The held pre-images accumulated up to this point will be flushed in case
@ -347,6 +356,16 @@ func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek
return pdb.StorageIterator(root, account, seek)
}
// IndexProgress returns the indexing progress made so far. It provides the
// number of states that remain unindexed.
func (db *Database) IndexProgress() (uint64, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return 0, errors.New("not supported")
}
return pdb.IndexProgress()
}
// IsVerkle returns the indicator if the database is holding a verkle tree.
func (db *Database) IsVerkle() bool {
return db.config.IsVerkle

9
triedb/pathdb/database.go
View file

@ -700,6 +700,15 @@ func (db *Database) HistoryRange() (uint64, uint64, error) {
return historyRange(db.freezer)
}
// IndexProgress returns the indexing progress made so far. It provides the
// number of states that remain unindexed.
func (db *Database) IndexProgress() (uint64, error) {
if db.indexer == nil {
return 0, nil
}
return db.indexer.progress()
}
// AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash.
func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {

64
triedb/pathdb/history_indexer.go
View file

@ -130,6 +130,8 @@ func (b *batchIndexer) finish(force bool) error {
var (
batch = b.db.NewBatch()
batchMu sync.RWMutex
storages int
start = time.Now()
eg errgroup.Group
)
eg.SetLimit(runtime.NumCPU())
@ -167,6 +169,7 @@ func (b *batchIndexer) finish(force bool) error {
})
}
for addrHash, slots := range b.storages {
storages += len(slots)
for storageHash, idList := range slots {
eg.Go(func() error {
if !b.delete {
@ -216,6 +219,7 @@ func (b *batchIndexer) finish(force bool) error {
if err := batch.Write(); err != nil {
return err
}
log.Debug("Committed batch indexer", "accounts", len(b.accounts), "storages", storages, "records", b.counter, "elapsed", common.PrettyDuration(time.Since(start)))
b.counter = 0
b.accounts = make(map[common.Hash][]uint64)
b.storages = make(map[common.Hash]map[common.Hash][]uint64)
@ -224,9 +228,10 @@ func (b *batchIndexer) finish(force bool) error {
// indexSingle processes the state history with the specified ID for indexing.
func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader) error {
defer func(start time.Time) {
start := time.Now()
defer func() {
indexHistoryTimer.UpdateSince(start)
}(time.Now())
}()
metadata := loadIndexMetadata(db)
if metadata == nil || metadata.Last+1 != historyID {
@ -247,15 +252,16 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient
if err := b.finish(true); err != nil {
return err
}
log.Debug("Indexed state history", "id", historyID)
log.Debug("Indexed state history", "id", historyID, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
// unindexSingle processes the state history with the specified ID for unindexing.
func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader) error {
defer func(start time.Time) {
start := time.Now()
defer func() {
unindexHistoryTimer.UpdateSince(start)
}(time.Now())
}()
metadata := loadIndexMetadata(db)
if metadata == nil || metadata.Last != historyID {
@ -276,7 +282,7 @@ func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancie
if err := b.finish(true); err != nil {
return err
}
log.Debug("Unindexed state history", "id", historyID)
log.Debug("Unindexed state history", "id", historyID, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
@ -299,6 +305,11 @@ type indexIniter struct {
interrupt chan *interruptSignal
done chan struct{}
closed chan struct{}
// indexing progress
indexed atomic.Uint64 // the id of latest indexed state
last atomic.Uint64 // the id of the target state to be indexed
wg sync.WaitGroup
}
@ -310,6 +321,14 @@ func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastID
done: make(chan struct{}),
closed: make(chan struct{}),
}
// Load indexing progress
initer.last.Store(lastID)
metadata := loadIndexMetadata(disk)
if metadata != nil {
initer.indexed.Store(metadata.Last)
}
// Launch background indexer
initer.wg.Add(1)
go initer.run(lastID)
return initer
@ -336,6 +355,22 @@ func (i *indexIniter) inited() bool {
}
}
func (i *indexIniter) remain() uint64 {
select {
case <-i.closed:
return 0
case <-i.done:
return 0
default:
last, indexed := i.last.Load(), i.indexed.Load()
if last < indexed {
log.Error("Invalid state indexing range", "last", last, "indexed", indexed)
return 0
}
return last - indexed
}
}
func (i *indexIniter) run(lastID uint64) {
defer i.wg.Done()
@ -361,6 +396,8 @@ func (i *indexIniter) run(lastID uint64) {
signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, signal.newLastID)
continue
}
i.last.Store(signal.newLastID) // update indexing range
// The index limit is extended by one, update the limit without
// interrupting the current background process.
if signal.newLastID == lastID+1 {
@ -498,9 +535,11 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
)
// Override the ETA if larger than the largest until now
eta := time.Duration(left/speed) * time.Millisecond
log.Info("Indexing state history", "processed", done, "left", left, "eta", common.PrettyDuration(eta))
log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
}
}
i.indexed.Store(current - 1) // update indexing progress
// Check interruption signal and abort process if it's fired
if interrupt != nil {
if signal := interrupt.Load(); signal != 0 {
@ -611,3 +650,14 @@ func (i *historyIndexer) shorten(historyID uint64) error {
return <-signal.result
}
}
// progress returns the indexing progress made so far. It provides the number
// of states that remain unindexed.
func (i *historyIndexer) progress() (uint64, error) {
select {
case <-i.initer.closed:
return 0, errors.New("indexer is closed")
default:
return i.initer.remain(), nil
}
}

63
triedb/pathdb/iterator_test.go
View file

@ -816,7 +816,8 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
config := &Config{
NoAsyncGeneration: true,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
memoryDB := rawdb.NewMemoryDatabase()
db := New(memoryDB, config, false)
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -831,6 +832,7 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0x33", "0x44", "0x55"), nil, nil, nil, false))
verify := func() {
// The output should be 11,33,44,55
it := newIterator(db, common.HexToHash("0x04"), common.Hash{})
// Do a quick check
@ -850,12 +852,35 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
}
}
}
verify()
if err := db.Journal(common.HexToHash("0x04")); err != nil {
t.Fatalf("Failed to journal the database, %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("Failed to close the database, %v", err)
}
db = New(memoryDB, config, false)
verify()
}
func TestStorageIteratorDeletions(t *testing.T) {
config := &Config{
NoAsyncGeneration: true,
}
db := New(rawdb.NewMemoryDatabase(), config, false)
memoryDB := rawdb.NewMemoryDatabase()
db := New(memoryDB, config, false)
restart := func(head common.Hash) {
if err := db.Journal(head); err != nil {
t.Fatalf("Failed to journal the database, %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("Failed to close the database, %v", err)
}
db = New(memoryDB, config, false)
}
// Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -874,6 +899,19 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x03"))
// The output should be 02,04,05,06
it, _ = db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 4, it, verifyStorage)
it.Release()
// The output should be 04,05,06
it, _ = db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.HexToHash("0x03"))
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Destruct the whole storage
accounts := map[common.Hash][]byte{
common.HexToHash("0xaa"): nil,
@ -885,6 +923,12 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 0, it, verifyStorage)
it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x04"))
it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 0, it, verifyStorage)
it.Release()
// Re-insert the slots of the same account
db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 4, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil), nil, nil, false))
@ -894,6 +938,14 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x05"))
// The output should be 07,08,09
it, _ = db.StorageIterator(common.HexToHash("0x05"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Destruct the whole storage but re-create the account in the same layer
db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 5, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}}), nil, nil, false))
@ -903,6 +955,13 @@ func TestStorageIteratorDeletions(t *testing.T) {
it.Release()
verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage)
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x06"))
it, _ = db.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12
it.Release()
verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage)
}
// TestStaleIterator tests if the iterator could correctly terminate the iteration

23
triedb/pathdb/states.go
View file

@ -387,8 +387,8 @@ func (s *stateSet) decode(r *rlp.Stream) error {
if err := r.Decode(&dec); err != nil {
return fmt.Errorf("load diff accounts: %v", err)
}
for i := 0; i < len(dec.AddrHashes); i++ {
accountSet[dec.AddrHashes[i]] = dec.Accounts[i]
for i := range dec.AddrHashes {
accountSet[dec.AddrHashes[i]] = empty2nil(dec.Accounts[i])
}
s.accountData = accountSet
@ -407,8 +407,8 @@ func (s *stateSet) decode(r *rlp.Stream) error {
}
for _, entry := range storages {
storageSet[entry.AddrHash] = make(map[common.Hash][]byte, len(entry.Keys))
for i := 0; i < len(entry.Keys); i++ {
storageSet[entry.AddrHash][entry.Keys[i]] = entry.Vals[i]
for i := range entry.Keys {
storageSet[entry.AddrHash][entry.Keys[i]] = empty2nil(entry.Vals[i])
}
}
s.storageData = storageSet
@ -550,8 +550,8 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error {
if err := r.Decode(&accounts); err != nil {
return fmt.Errorf("load diff account origin set: %v", err)
}
for i := 0; i < len(accounts.Accounts); i++ {
accountSet[accounts.Addresses[i]] = accounts.Accounts[i]
for i := range accounts.Accounts {
accountSet[accounts.Addresses[i]] = empty2nil(accounts.Accounts[i])
}
s.accountOrigin = accountSet
@ -570,10 +570,17 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error {
}
for _, storage := range storages {
storageSet[storage.Address] = make(map[common.Hash][]byte)
for i := 0; i < len(storage.Keys); i++ {
storageSet[storage.Address][storage.Keys[i]] = storage.Vals[i]
for i := range storage.Keys {
storageSet[storage.Address][storage.Keys[i]] = empty2nil(storage.Vals[i])
}
}
s.storageOrigin = storageSet
return nil
}
func empty2nil(b []byte) []byte {
if len(b) == 0 {
return nil
}
return b
}

4
version/version.go
View file

@ -18,7 +18,7 @@ package version
const (
Major = 1 // Major version component of the current release
Minor = 15 // Minor version component of the current release
Patch = 12 // Patch version component of the current release
Minor = 16 // Minor version component of the current release
Patch = 1 // Patch version component of the current release
Meta = "unstable" // Version metadata to append to the version string
)