Merge branch 'ethereum:master' into master

This commit is contained in:
juga1980 2025-06-27 12:21:04 +02:00 committed by GitHub
commit f221f05b61
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 208 additions and 142 deletions

View file

@ -2,8 +2,8 @@ on:
schedule: schedule:
- cron: '0 16 * * *' - cron: '0 16 * * *'
push: push:
branches: tags:
- "release/*" - "v*"
workflow_dispatch: workflow_dispatch:
jobs: jobs:

View file

@ -1,86 +0,0 @@
language: go
go_import_path: github.com/ethereum/go-ethereum
sudo: false
jobs:
include:
# This builder create and push the Docker images for all architectures
- stage: build
if: type = push && tag ~= /^v[0-9]/
os: linux
arch: amd64
dist: focal
go: 1.24.x
env:
- docker
services:
- docker
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- export DOCKER_CLI_EXPERIMENTAL=enabled
script:
- go run build/ci.go dockerx -platform "linux/amd64,linux/arm64,linux/riscv64" -hub ethereum/client-go -upload
# This builder does the Ubuntu PPA nightly uploads
- stage: build
if: type = push && tag ~= /^v[0-9]/
os: linux
dist: focal
go: 1.24.x
env:
- ubuntu-ppa
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- stage: build
if: type = push && tag ~= /^v[0-9]/
os: linux
dist: focal
sudo: required
go: 1.24.x
env:
- azure-linux
git:
submodules: false # avoid cloning ethereum/tests
script:
# build amd64
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# build 386
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib
- git status --porcelain
- go run build/ci.go install -dlgo -arch 386
- go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# Switch over GCC to cross compilation (breaks 386, hence why do it here only)
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo ln -s /usr/include/asm-generic /usr/include/asm
- GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc
- GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# This builder does the Azure archive purges to avoid accumulating junk
- stage: build
if: type = cron
os: linux
dist: focal
go: 1.24.x
env:
- azure-purge
git:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go purge -store gethstore/builds -days 14

View file

@ -58,7 +58,7 @@ var (
ArgsUsage: "<genesisPath>", ArgsUsage: "<genesisPath>",
Flags: slices.Concat([]cli.Flag{ Flags: slices.Concat([]cli.Flag{
utils.CachePreimagesFlag, utils.CachePreimagesFlag,
utils.OverridePrague, utils.OverrideOsaka,
utils.OverrideVerkle, utils.OverrideVerkle,
}, utils.DatabaseFlags), }, utils.DatabaseFlags),
Description: ` Description: `
@ -269,9 +269,9 @@ func initGenesis(ctx *cli.Context) error {
defer stack.Close() defer stack.Close()
var overrides core.ChainOverrides var overrides core.ChainOverrides
if ctx.IsSet(utils.OverridePrague.Name) { if ctx.IsSet(utils.OverrideOsaka.Name) {
v := ctx.Uint64(utils.OverridePrague.Name) v := ctx.Uint64(utils.OverrideOsaka.Name)
overrides.OverridePrague = &v overrides.OverrideOsaka = &v
} }
if ctx.IsSet(utils.OverrideVerkle.Name) { if ctx.IsSet(utils.OverrideVerkle.Name) {
v := ctx.Uint64(utils.OverrideVerkle.Name) v := ctx.Uint64(utils.OverrideVerkle.Name)

View file

@ -223,9 +223,9 @@ func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string {
// makeFullNode loads geth configuration and creates the Ethereum backend. // makeFullNode loads geth configuration and creates the Ethereum backend.
func makeFullNode(ctx *cli.Context) *node.Node { func makeFullNode(ctx *cli.Context) *node.Node {
stack, cfg := makeConfigNode(ctx) stack, cfg := makeConfigNode(ctx)
if ctx.IsSet(utils.OverridePrague.Name) { if ctx.IsSet(utils.OverrideOsaka.Name) {
v := ctx.Uint64(utils.OverridePrague.Name) v := ctx.Uint64(utils.OverrideOsaka.Name)
cfg.Eth.OverridePrague = &v cfg.Eth.OverrideOsaka = &v
} }
if ctx.IsSet(utils.OverrideVerkle.Name) { if ctx.IsSet(utils.OverrideVerkle.Name) {
v := ctx.Uint64(utils.OverrideVerkle.Name) v := ctx.Uint64(utils.OverrideVerkle.Name)

View file

@ -62,7 +62,7 @@ var (
utils.NoUSBFlag, // deprecated utils.NoUSBFlag, // deprecated
utils.USBFlag, utils.USBFlag,
utils.SmartCardDaemonPathFlag, utils.SmartCardDaemonPathFlag,
utils.OverridePrague, utils.OverrideOsaka,
utils.OverrideVerkle, utils.OverrideVerkle,
utils.EnablePersonal, // deprecated utils.EnablePersonal, // deprecated
utils.TxPoolLocalsFlag, utils.TxPoolLocalsFlag,

View file

@ -243,9 +243,9 @@ var (
Value: 2048, Value: 2048,
Category: flags.EthCategory, Category: flags.EthCategory,
} }
OverridePrague = &cli.Uint64Flag{ OverrideOsaka = &cli.Uint64Flag{
Name: "override.prague", Name: "override.osaka",
Usage: "Manually specify the Prague fork timestamp, overriding the bundled setting", Usage: "Manually specify the Osaka fork timestamp, overriding the bundled setting",
Category: flags.EthCategory, Category: flags.EthCategory,
} }
OverrideVerkle = &cli.Uint64Flag{ OverrideVerkle = &cli.Uint64Flag{

View file

@ -426,6 +426,11 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) {
return bc.txIndexer.txIndexProgress(), nil return bc.txIndexer.txIndexProgress(), nil
} }
// StateIndexProgress returns the historical state indexing progress.
func (bc *BlockChain) StateIndexProgress() (uint64, error) {
return bc.triedb.IndexProgress()
}
// HistoryPruningCutoff returns the configured history pruning point. // HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database. // Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) { func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) {

View file

@ -258,7 +258,7 @@ func (e *GenesisMismatchError) Error() string {
// ChainOverrides contains the changes to chain config. // ChainOverrides contains the changes to chain config.
type ChainOverrides struct { type ChainOverrides struct {
OverridePrague *uint64 OverrideOsaka *uint64
OverrideVerkle *uint64 OverrideVerkle *uint64
} }
@ -267,8 +267,8 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error {
if o == nil || cfg == nil { if o == nil || cfg == nil {
return nil return nil
} }
if o.OverridePrague != nil { if o.OverrideOsaka != nil {
cfg.PragueTime = o.OverridePrague cfg.OsakaTime = o.OverrideOsaka
} }
if o.OverrideVerkle != nil { if o.OverrideVerkle != nil {
cfg.VerkleTime = o.OverrideVerkle cfg.VerkleTime = o.OverrideVerkle

View file

@ -403,6 +403,10 @@ func (b *EthAPIBackend) SyncProgress(ctx context.Context) ethereum.SyncProgress
prog.TxIndexFinishedBlocks = txProg.Indexed prog.TxIndexFinishedBlocks = txProg.Indexed
prog.TxIndexRemainingBlocks = txProg.Remaining prog.TxIndexRemainingBlocks = txProg.Remaining
} }
remain, err := b.eth.blockchain.StateIndexProgress()
if err == nil {
prog.StateIndexRemaining = remain
}
return prog return prog
} }

View file

@ -253,8 +253,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
} }
// Override the chain config with provided settings. // Override the chain config with provided settings.
var overrides core.ChainOverrides var overrides core.ChainOverrides
if config.OverridePrague != nil { if config.OverrideOsaka != nil {
overrides.OverridePrague = config.OverridePrague overrides.OverrideOsaka = config.OverrideOsaka
} }
if config.OverrideVerkle != nil { if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle overrides.OverrideVerkle = config.OverrideVerkle

View file

@ -81,6 +81,10 @@ func (api *DownloaderAPI) eventLoop() {
prog.TxIndexFinishedBlocks = txProg.Indexed prog.TxIndexFinishedBlocks = txProg.Indexed
prog.TxIndexRemainingBlocks = txProg.Remaining prog.TxIndexRemainingBlocks = txProg.Remaining
} }
remain, err := api.chain.StateIndexProgress()
if err == nil {
prog.StateIndexRemaining = remain
}
return prog return prog
} }
) )

View file

@ -158,8 +158,8 @@ type Config struct {
// send-transaction variants. The unit is ether. // send-transaction variants. The unit is ether.
RPCTxFeeCap float64 RPCTxFeeCap float64
// OverridePrague (TODO: remove after the fork) // OverrideOsaka (TODO: remove after the fork)
OverridePrague *uint64 `toml:",omitempty"` OverrideOsaka *uint64 `toml:",omitempty"`
// OverrideVerkle (TODO: remove after the fork) // OverrideVerkle (TODO: remove after the fork)
OverrideVerkle *uint64 `toml:",omitempty"` OverrideVerkle *uint64 `toml:",omitempty"`

View file

@ -54,7 +54,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
RPCGasCap uint64 RPCGasCap uint64
RPCEVMTimeout time.Duration RPCEVMTimeout time.Duration
RPCTxFeeCap float64 RPCTxFeeCap float64
OverridePrague *uint64 `toml:",omitempty"` OverrideOsaka *uint64 `toml:",omitempty"`
OverrideVerkle *uint64 `toml:",omitempty"` OverrideVerkle *uint64 `toml:",omitempty"`
} }
var enc Config var enc Config
@ -95,7 +95,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.RPCGasCap = c.RPCGasCap enc.RPCGasCap = c.RPCGasCap
enc.RPCEVMTimeout = c.RPCEVMTimeout enc.RPCEVMTimeout = c.RPCEVMTimeout
enc.RPCTxFeeCap = c.RPCTxFeeCap enc.RPCTxFeeCap = c.RPCTxFeeCap
enc.OverridePrague = c.OverridePrague enc.OverrideOsaka = c.OverrideOsaka
enc.OverrideVerkle = c.OverrideVerkle enc.OverrideVerkle = c.OverrideVerkle
return &enc, nil return &enc, nil
} }
@ -140,7 +140,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
RPCGasCap *uint64 RPCGasCap *uint64
RPCEVMTimeout *time.Duration RPCEVMTimeout *time.Duration
RPCTxFeeCap *float64 RPCTxFeeCap *float64
OverridePrague *uint64 `toml:",omitempty"` OverrideOsaka *uint64 `toml:",omitempty"`
OverrideVerkle *uint64 `toml:",omitempty"` OverrideVerkle *uint64 `toml:",omitempty"`
} }
var dec Config var dec Config
@ -258,8 +258,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.RPCTxFeeCap != nil { if dec.RPCTxFeeCap != nil {
c.RPCTxFeeCap = *dec.RPCTxFeeCap c.RPCTxFeeCap = *dec.RPCTxFeeCap
} }
if dec.OverridePrague != nil { if dec.OverrideOsaka != nil {
c.OverridePrague = dec.OverridePrague c.OverrideOsaka = dec.OverrideOsaka
} }
if dec.OverrideVerkle != nil { if dec.OverrideVerkle != nil {
c.OverrideVerkle = dec.OverrideVerkle c.OverrideVerkle = dec.OverrideVerkle

View file

@ -789,6 +789,7 @@ type rpcProgress struct {
HealingBytecode hexutil.Uint64 HealingBytecode hexutil.Uint64
TxIndexFinishedBlocks hexutil.Uint64 TxIndexFinishedBlocks hexutil.Uint64
TxIndexRemainingBlocks hexutil.Uint64 TxIndexRemainingBlocks hexutil.Uint64
StateIndexRemaining hexutil.Uint64
} }
func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress {
@ -815,5 +816,6 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress {
HealingBytecode: uint64(p.HealingBytecode), HealingBytecode: uint64(p.HealingBytecode),
TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks), TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks),
TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks), TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks),
StateIndexRemaining: uint64(p.StateIndexRemaining),
} }
} }

View file

@ -1510,6 +1510,9 @@ func (s *SyncState) TxIndexFinishedBlocks() hexutil.Uint64 {
func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 { func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 {
return hexutil.Uint64(s.progress.TxIndexRemainingBlocks) return hexutil.Uint64(s.progress.TxIndexRemainingBlocks)
} }
func (s *SyncState) StateIndexRemaining() hexutil.Uint64 {
return hexutil.Uint64(s.progress.StateIndexRemaining)
}
// Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not // Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not
// yet received the latest block headers from its peers. In case it is synchronizing: // yet received the latest block headers from its peers. In case it is synchronizing:

View file

@ -124,6 +124,9 @@ type SyncProgress struct {
// "transaction indexing" fields // "transaction indexing" fields
TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed
TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet
// "historical state indexing" fields
StateIndexRemaining uint64 // Number of states remain unindexed
} }
// Done returns the indicator if the initial sync is finished or not. // Done returns the indicator if the initial sync is finished or not.
@ -131,7 +134,7 @@ func (prog SyncProgress) Done() bool {
if prog.CurrentBlock < prog.HighestBlock { if prog.CurrentBlock < prog.HighestBlock {
return false return false
} }
return prog.TxIndexRemainingBlocks == 0 return prog.TxIndexRemainingBlocks == 0 && prog.StateIndexRemaining == 0
} }
// ChainSyncReader wraps access to the node's current sync status. If there's no // ChainSyncReader wraps access to the node's current sync status. If there's no

View file

@ -170,6 +170,7 @@ func (api *EthereumAPI) Syncing(ctx context.Context) (interface{}, error) {
"healingBytecode": hexutil.Uint64(progress.HealingBytecode), "healingBytecode": hexutil.Uint64(progress.HealingBytecode),
"txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks), "txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks),
"txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks), "txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks),
"stateIndexRemaining": hexutil.Uint64(progress.StateIndexRemaining),
}, nil }, nil
} }

View file

@ -3977,6 +3977,7 @@ var outputSyncingFormatter = function(result) {
result.healingBytecode = utils.toDecimal(result.healingBytecode); result.healingBytecode = utils.toDecimal(result.healingBytecode);
result.txIndexFinishedBlocks = utils.toDecimal(result.txIndexFinishedBlocks); result.txIndexFinishedBlocks = utils.toDecimal(result.txIndexFinishedBlocks);
result.txIndexRemainingBlocks = utils.toDecimal(result.txIndexRemainingBlocks); result.txIndexRemainingBlocks = utils.toDecimal(result.txIndexRemainingBlocks);
result.stateIndexRemaining = utils.toDecimal(result.stateIndexRemaining)
return result; return result;
}; };

View file

@ -109,7 +109,7 @@ func newLevelDBDatabase(file string, cache int, handles int, namespace string, r
return nil, err return nil, err
} }
log.Info("Using LevelDB as the backing database") log.Info("Using LevelDB as the backing database")
return rawdb.NewDatabase(db), nil return db, nil
} }
// newPebbleDBDatabase creates a persistent key-value database without a freezer // newPebbleDBDatabase creates a persistent key-value database without a freezer
@ -119,5 +119,5 @@ func newPebbleDBDatabase(file string, cache int, handles int, namespace string,
if err != nil { if err != nil {
return nil, err return nil, err
} }
return rawdb.NewDatabase(db), nil return db, nil
} }

View file

@ -356,6 +356,16 @@ func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek
return pdb.StorageIterator(root, account, seek) return pdb.StorageIterator(root, account, seek)
} }
// IndexProgress returns the indexing progress made so far. It provides the
// number of states that remain unindexed.
func (db *Database) IndexProgress() (uint64, error) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return 0, errors.New("not supported")
}
return pdb.IndexProgress()
}
// IsVerkle returns the indicator if the database is holding a verkle tree. // IsVerkle returns the indicator if the database is holding a verkle tree.
func (db *Database) IsVerkle() bool { func (db *Database) IsVerkle() bool {
return db.config.IsVerkle return db.config.IsVerkle

View file

@ -700,6 +700,15 @@ func (db *Database) HistoryRange() (uint64, uint64, error) {
return historyRange(db.freezer) return historyRange(db.freezer)
} }
// IndexProgress returns the indexing progress made so far. It provides the
// number of states that remain unindexed.
func (db *Database) IndexProgress() (uint64, error) {
if db.indexer == nil {
return 0, nil
}
return db.indexer.progress()
}
// AccountIterator creates a new account iterator for the specified root hash and // AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash. // seeks to a starting account hash.
func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) { func (db *Database) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {

View file

@ -305,6 +305,11 @@ type indexIniter struct {
interrupt chan *interruptSignal interrupt chan *interruptSignal
done chan struct{} done chan struct{}
closed chan struct{} closed chan struct{}
// indexing progress
indexed atomic.Uint64 // the id of latest indexed state
last atomic.Uint64 // the id of the target state to be indexed
wg sync.WaitGroup wg sync.WaitGroup
} }
@ -316,6 +321,14 @@ func newIndexIniter(disk ethdb.KeyValueStore, freezer ethdb.AncientStore, lastID
done: make(chan struct{}), done: make(chan struct{}),
closed: make(chan struct{}), closed: make(chan struct{}),
} }
// Load indexing progress
initer.last.Store(lastID)
metadata := loadIndexMetadata(disk)
if metadata != nil {
initer.indexed.Store(metadata.Last)
}
// Launch background indexer
initer.wg.Add(1) initer.wg.Add(1)
go initer.run(lastID) go initer.run(lastID)
return initer return initer
@ -342,6 +355,22 @@ func (i *indexIniter) inited() bool {
} }
} }
func (i *indexIniter) remain() uint64 {
select {
case <-i.closed:
return 0
case <-i.done:
return 0
default:
last, indexed := i.last.Load(), i.indexed.Load()
if last < indexed {
log.Error("Invalid state indexing range", "last", last, "indexed", indexed)
return 0
}
return last - indexed
}
}
func (i *indexIniter) run(lastID uint64) { func (i *indexIniter) run(lastID uint64) {
defer i.wg.Done() defer i.wg.Done()
@ -367,6 +396,8 @@ func (i *indexIniter) run(lastID uint64) {
signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, signal.newLastID) signal.result <- fmt.Errorf("invalid history id, last: %d, got: %d", lastID, signal.newLastID)
continue continue
} }
i.last.Store(signal.newLastID) // update indexing range
// The index limit is extended by one, update the limit without // The index limit is extended by one, update the limit without
// interrupting the current background process. // interrupting the current background process.
if signal.newLastID == lastID+1 { if signal.newLastID == lastID+1 {
@ -507,6 +538,8 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
} }
} }
i.indexed.Store(current - 1) // update indexing progress
// Check interruption signal and abort process if it's fired // Check interruption signal and abort process if it's fired
if interrupt != nil { if interrupt != nil {
if signal := interrupt.Load(); signal != 0 { if signal := interrupt.Load(); signal != 0 {
@ -617,3 +650,14 @@ func (i *historyIndexer) shorten(historyID uint64) error {
return <-signal.result return <-signal.result
} }
} }
// progress returns the indexing progress made so far. It provides the number
// of states that remain unindexed.
func (i *historyIndexer) progress() (uint64, error) {
select {
case <-i.initer.closed:
return 0, errors.New("indexer is closed")
default:
return i.initer.remain(), nil
}
}

View file

@ -816,7 +816,8 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
config := &Config{ config := &Config{
NoAsyncGeneration: true, NoAsyncGeneration: true,
} }
db := New(rawdb.NewMemoryDatabase(), config, false) memoryDB := rawdb.NewMemoryDatabase()
db := New(memoryDB, config, false)
// Stack three diff layers on top with various overlaps // Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -831,6 +832,7 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(), db.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), 3, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0x33", "0x44", "0x55"), nil, nil, nil, false)) NewStateSetWithOrigin(randomAccountSet("0x33", "0x44", "0x55"), nil, nil, nil, false))
verify := func() {
// The output should be 11,33,44,55 // The output should be 11,33,44,55
it := newIterator(db, common.HexToHash("0x04"), common.Hash{}) it := newIterator(db, common.HexToHash("0x04"), common.Hash{})
// Do a quick check // Do a quick check
@ -849,13 +851,36 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(db *Database, r
t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) t.Errorf("expected deleted elem %x to not be returned by iterator", deleted)
} }
} }
}
verify()
if err := db.Journal(common.HexToHash("0x04")); err != nil {
t.Fatalf("Failed to journal the database, %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("Failed to close the database, %v", err)
}
db = New(memoryDB, config, false)
verify()
} }
func TestStorageIteratorDeletions(t *testing.T) { func TestStorageIteratorDeletions(t *testing.T) {
config := &Config{ config := &Config{
NoAsyncGeneration: true, NoAsyncGeneration: true,
} }
db := New(rawdb.NewMemoryDatabase(), config, false) memoryDB := rawdb.NewMemoryDatabase()
db := New(memoryDB, config, false)
restart := func(head common.Hash) {
if err := db.Journal(head); err != nil {
t.Fatalf("Failed to journal the database, %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("Failed to close the database, %v", err)
}
db = New(memoryDB, config, false)
}
// Stack three diff layers on top with various overlaps // Stack three diff layers on top with various overlaps
db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(), db.Update(common.HexToHash("0x02"), types.EmptyRootHash, 1, trienode.NewMergedNodeSet(),
@ -874,6 +899,19 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 3, it, verifyStorage) verifyIterator(t, 3, it, verifyStorage)
it.Release() it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x03"))
// The output should be 02,04,05,06
it, _ = db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 4, it, verifyStorage)
it.Release()
// The output should be 04,05,06
it, _ = db.StorageIterator(common.HexToHash("0x03"), common.HexToHash("0xaa"), common.HexToHash("0x03"))
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Destruct the whole storage // Destruct the whole storage
accounts := map[common.Hash][]byte{ accounts := map[common.Hash][]byte{
common.HexToHash("0xaa"): nil, common.HexToHash("0xaa"): nil,
@ -885,6 +923,12 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 0, it, verifyStorage) verifyIterator(t, 0, it, verifyStorage)
it.Release() it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x04"))
it, _ = db.StorageIterator(common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 0, it, verifyStorage)
it.Release()
// Re-insert the slots of the same account // Re-insert the slots of the same account
db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 4, trienode.NewMergedNodeSet(), db.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), 4, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil), nil, nil, false)) NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil), nil, nil, false))
@ -894,6 +938,14 @@ func TestStorageIteratorDeletions(t *testing.T) {
verifyIterator(t, 3, it, verifyStorage) verifyIterator(t, 3, it, verifyStorage)
it.Release() it.Release()
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x05"))
// The output should be 07,08,09
it, _ = db.StorageIterator(common.HexToHash("0x05"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 3, it, verifyStorage)
it.Release()
// Destruct the whole storage but re-create the account in the same layer // Destruct the whole storage but re-create the account in the same layer
db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 5, trienode.NewMergedNodeSet(), db.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), 5, trienode.NewMergedNodeSet(),
NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}}), nil, nil, false)) NewStateSetWithOrigin(randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}}), nil, nil, false))
@ -903,6 +955,13 @@ func TestStorageIteratorDeletions(t *testing.T) {
it.Release() it.Release()
verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage)
// Ensure the iteration result aligns after the database restart
restart(common.HexToHash("0x06"))
it, _ = db.StorageIterator(common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{})
verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12
it.Release()
verifyIterator(t, 2, db.tree.get(common.HexToHash("0x06")).(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage)
} }
// TestStaleIterator tests if the iterator could correctly terminate the iteration // TestStaleIterator tests if the iterator could correctly terminate the iteration

View file

@ -387,8 +387,8 @@ func (s *stateSet) decode(r *rlp.Stream) error {
if err := r.Decode(&dec); err != nil { if err := r.Decode(&dec); err != nil {
return fmt.Errorf("load diff accounts: %v", err) return fmt.Errorf("load diff accounts: %v", err)
} }
for i := 0; i < len(dec.AddrHashes); i++ { for i := range dec.AddrHashes {
accountSet[dec.AddrHashes[i]] = dec.Accounts[i] accountSet[dec.AddrHashes[i]] = empty2nil(dec.Accounts[i])
} }
s.accountData = accountSet s.accountData = accountSet
@ -407,8 +407,8 @@ func (s *stateSet) decode(r *rlp.Stream) error {
} }
for _, entry := range storages { for _, entry := range storages {
storageSet[entry.AddrHash] = make(map[common.Hash][]byte, len(entry.Keys)) storageSet[entry.AddrHash] = make(map[common.Hash][]byte, len(entry.Keys))
for i := 0; i < len(entry.Keys); i++ { for i := range entry.Keys {
storageSet[entry.AddrHash][entry.Keys[i]] = entry.Vals[i] storageSet[entry.AddrHash][entry.Keys[i]] = empty2nil(entry.Vals[i])
} }
} }
s.storageData = storageSet s.storageData = storageSet
@ -550,8 +550,8 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error {
if err := r.Decode(&accounts); err != nil { if err := r.Decode(&accounts); err != nil {
return fmt.Errorf("load diff account origin set: %v", err) return fmt.Errorf("load diff account origin set: %v", err)
} }
for i := 0; i < len(accounts.Accounts); i++ { for i := range accounts.Accounts {
accountSet[accounts.Addresses[i]] = accounts.Accounts[i] accountSet[accounts.Addresses[i]] = empty2nil(accounts.Accounts[i])
} }
s.accountOrigin = accountSet s.accountOrigin = accountSet
@ -570,10 +570,17 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error {
} }
for _, storage := range storages { for _, storage := range storages {
storageSet[storage.Address] = make(map[common.Hash][]byte) storageSet[storage.Address] = make(map[common.Hash][]byte)
for i := 0; i < len(storage.Keys); i++ { for i := range storage.Keys {
storageSet[storage.Address][storage.Keys[i]] = storage.Vals[i] storageSet[storage.Address][storage.Keys[i]] = empty2nil(storage.Vals[i])
} }
} }
s.storageOrigin = storageSet s.storageOrigin = storageSet
return nil return nil
} }
func empty2nil(b []byte) []byte {
if len(b) == 0 {
return nil
}
return b
}

View file

@ -18,7 +18,7 @@ package version
const ( const (
Major = 1 // Major version component of the current release Major = 1 // Major version component of the current release
Minor = 15 // Minor version component of the current release Minor = 16 // Minor version component of the current release
Patch = 12 // Patch version component of the current release Patch = 1 // Patch version component of the current release
Meta = "unstable" // Version metadata to append to the version string Meta = "unstable" // Version metadata to append to the version string
) )