all: improve ETA calculation across all progress indicators (#32521)
Some checks are pending
/ Linux Build (push) Waiting to run
/ Linux Build (arm) (push) Waiting to run
/ Windows Build (push) Waiting to run
/ Docker Image (push) Waiting to run

### Summary
Fixes long-standing ETA calculation errors in progress indicators that
have been present since February 2021. The current implementation
produces increasingly inaccurate estimates due to integer division
precision loss.

### Problem

3aeccadd04/triedb/pathdb/history_indexer.go (L541-L553)
The ETA calculation has two critical issues:
1. **Integer division precision loss**: `speed` is calculated as
`uint64`
2. **Off-by-one**: `speed` uses `+ 1`(2 times) to avoid division by
zero, however it makes mistake in the final calculation

This results in wildly inaccurate time estimates that don't improve as
progress continues.

### Example
Current output during state history indexing:
```
lvl=info msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s
```

**Expected calculation:**
- Speed: 16858580 ÷ 66179848ms = 0.255 blocks/ms  
- ETA: 41802252 ÷ 0.255 = ~45.6 hours

**Current buggy calculation:**
- Speed: rounds to 1 block/ms
- ETA: 41802252 ÷ 1 = ~11.6 hours 

### Solution
- Created centralized `CalculateETA()` function in common package
- Replaced all 8 duplicate code copies across the codebase

### Testing
Verified accurate ETA calculations during archive node reindexing with
significantly improved time estimates.
This commit is contained in:
Mars 2025-08-31 23:47:02 -06:00 committed by GitHub
parent 0cde5278e8
commit 0e69530c6e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 104 additions and 26 deletions

30
common/eta.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package common
import "time"
// CalculateETA calculates the estimated remaining time based on the
// number of finished task, remaining task, and the time cost for finished task.
func CalculateETA(done, left uint64, elapsed time.Duration) time.Duration {
if done == 0 || elapsed.Milliseconds() == 0 {
return 0
}
speed := float64(done) / float64(elapsed.Milliseconds())
return time.Duration(float64(left)/speed) * time.Millisecond
}

60
common/eta_test.go Normal file
View file

@ -0,0 +1,60 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package common
import (
"testing"
"time"
)
func TestCalculateETA(t *testing.T) {
type args struct {
done uint64
left uint64
elapsed time.Duration
}
tests := []struct {
name string
args args
want time.Duration
}{
{
name: "zero done",
args: args{done: 0, left: 100, elapsed: time.Second},
want: 0,
},
{
name: "zero elapsed",
args: args{done: 1, left: 100, elapsed: 0},
want: 0,
},
{
name: "@Jolly23 's case",
args: args{done: 16858580, left: 41802252, elapsed: 66179848 * time.Millisecond},
want: 164098440 * time.Millisecond,
// wrong msg: msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s
// should be around 45.58 hours
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := CalculateETA(tt.args.done, tt.args.left, tt.args.elapsed); got != tt.want {
t.Errorf("CalculateETA() = %v ms, want %v ms", got.Milliseconds(), tt.want)
}
})
}
}

View file

@ -160,11 +160,8 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
var eta time.Duration // Realistically will never remain uninited
if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
var (
left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
)
eta = time.Duration(left/speed) * time.Millisecond
left := math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
eta = common.CalculateETA(done, left, time.Since(pstart))
}
if time.Since(logged) > 8*time.Second {
log.Info("Pruning state data", "nodes", count, "skipped", skipped, "size", size,

View file

@ -172,19 +172,15 @@ func (stat *generateStats) report() {
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
var (
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
eta = time.Duration(left/speed) * time.Millisecond
eta = common.CalculateETA(done, left, time.Since(stat.start))
)
// If there are large contract crawls in progress, estimate their finish time
for acc, head := range stat.slotsHead {
start := stat.slotsStart[acc]
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
var (
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
)
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
// Override the ETA if larger than the largest until now
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
eta = slotETA
}
}

View file

@ -545,10 +545,8 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
var (
left = lastID - current + 1
done = current - beginID
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
)
// Override the ETA if larger than the largest until now
eta := time.Duration(left/speed) * time.Millisecond
eta := common.CalculateETA(done, left, time.Since(start))
log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
}
}

View file

@ -167,19 +167,16 @@ func (stat *generateStats) report() {
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
var (
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
eta = time.Duration(left/speed) * time.Millisecond
eta = common.CalculateETA(done, left, time.Since(stat.start))
)
// If there are large contract crawls in progress, estimate their finish time
for acc, head := range stat.slotsHead {
start := stat.slotsStart[acc]
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
var (
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
)
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
// Override the ETA if larger than the largest until now
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
eta = slotETA
}
}