mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-02-26 07:37:20 +00:00
all: improve ETA calculation across all progress indicators (#32521)
### Summary
Fixes long-standing ETA calculation errors in progress indicators that
have been present since February 2021. The current implementation
produces increasingly inaccurate estimates due to integer division
precision loss.
### Problem
3aeccadd04/triedb/pathdb/history_indexer.go (L541-L553)
The ETA calculation has two critical issues:
1. **Integer division precision loss**: `speed` is calculated as
`uint64`
2. **Off-by-one**: `speed` uses `+ 1`(2 times) to avoid division by
zero, however it makes mistake in the final calculation
This results in wildly inaccurate time estimates that don't improve as
progress continues.
### Example
Current output during state history indexing:
```
lvl=info msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s
```
**Expected calculation:**
- Speed: 16858580 ÷ 66179848ms = 0.255 blocks/ms
- ETA: 41802252 ÷ 0.255 = ~45.6 hours
**Current buggy calculation:**
- Speed: rounds to 1 block/ms
- ETA: 41802252 ÷ 1 = ~11.6 hours ❌
### Solution
- Created centralized `CalculateETA()` function in common package
- Replaced all 8 duplicate code copies across the codebase
### Testing
Verified accurate ETA calculations during archive node reindexing with
significantly improved time estimates.
This commit is contained in:
parent
0cde5278e8
commit
0e69530c6e
6 changed files with 104 additions and 26 deletions
30
common/eta.go
Normal file
30
common/eta.go
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package common
|
||||
|
||||
import "time"
|
||||
|
||||
// CalculateETA calculates the estimated remaining time based on the
|
||||
// number of finished task, remaining task, and the time cost for finished task.
|
||||
func CalculateETA(done, left uint64, elapsed time.Duration) time.Duration {
|
||||
if done == 0 || elapsed.Milliseconds() == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
speed := float64(done) / float64(elapsed.Milliseconds())
|
||||
return time.Duration(float64(left)/speed) * time.Millisecond
|
||||
}
|
||||
60
common/eta_test.go
Normal file
60
common/eta_test.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCalculateETA(t *testing.T) {
|
||||
type args struct {
|
||||
done uint64
|
||||
left uint64
|
||||
elapsed time.Duration
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want time.Duration
|
||||
}{
|
||||
{
|
||||
name: "zero done",
|
||||
args: args{done: 0, left: 100, elapsed: time.Second},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "zero elapsed",
|
||||
args: args{done: 1, left: 100, elapsed: 0},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "@Jolly23 's case",
|
||||
args: args{done: 16858580, left: 41802252, elapsed: 66179848 * time.Millisecond},
|
||||
want: 164098440 * time.Millisecond,
|
||||
// wrong msg: msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s
|
||||
// should be around 45.58 hours
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := CalculateETA(tt.args.done, tt.args.left, tt.args.elapsed); got != tt.want {
|
||||
t.Errorf("CalculateETA() = %v ms, want %v ms", got.Milliseconds(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -160,11 +160,8 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
|
|||
|
||||
var eta time.Duration // Realistically will never remain uninited
|
||||
if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
|
||||
var (
|
||||
left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
|
||||
speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
)
|
||||
eta = time.Duration(left/speed) * time.Millisecond
|
||||
left := math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
|
||||
eta = common.CalculateETA(done, left, time.Since(pstart))
|
||||
}
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Pruning state data", "nodes", count, "skipped", skipped, "size", size,
|
||||
|
|
|
|||
|
|
@ -171,20 +171,16 @@ func (stat *generateStats) report() {
|
|||
// If there's progress on the account trie, estimate the time to finish crawling it
|
||||
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
|
||||
var (
|
||||
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
|
||||
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
eta = time.Duration(left/speed) * time.Millisecond
|
||||
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
|
||||
eta = common.CalculateETA(done, left, time.Since(stat.start))
|
||||
)
|
||||
// If there are large contract crawls in progress, estimate their finish time
|
||||
for acc, head := range stat.slotsHead {
|
||||
start := stat.slotsStart[acc]
|
||||
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
|
||||
var (
|
||||
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
|
||||
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
)
|
||||
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
|
||||
// Override the ETA if larger than the largest until now
|
||||
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
|
||||
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
|
||||
eta = slotETA
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -543,12 +543,10 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
|
|||
logged = time.Now()
|
||||
|
||||
var (
|
||||
left = lastID - current + 1
|
||||
done = current - beginID
|
||||
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
left = lastID - current + 1
|
||||
done = current - beginID
|
||||
)
|
||||
// Override the ETA if larger than the largest until now
|
||||
eta := time.Duration(left/speed) * time.Millisecond
|
||||
eta := common.CalculateETA(done, left, time.Since(start))
|
||||
log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,20 +166,17 @@ func (stat *generateStats) report() {
|
|||
// If there's progress on the account trie, estimate the time to finish crawling it
|
||||
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
|
||||
var (
|
||||
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
|
||||
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
eta = time.Duration(left/speed) * time.Millisecond
|
||||
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
|
||||
eta = common.CalculateETA(done, left, time.Since(stat.start))
|
||||
)
|
||||
// If there are large contract crawls in progress, estimate their finish time
|
||||
for acc, head := range stat.slotsHead {
|
||||
start := stat.slotsStart[acc]
|
||||
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
|
||||
var (
|
||||
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
|
||||
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
|
||||
)
|
||||
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
|
||||
|
||||
// Override the ETA if larger than the largest until now
|
||||
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
|
||||
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
|
||||
eta = slotETA
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue