mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-24 13:46:43 +00:00
no log
Signed-off-by: Delweng <delweng@gmail.com>
This commit is contained in:
parent
1c693226b8
commit
fcb657fff4
3 changed files with 16 additions and 82 deletions
|
|
@ -21,7 +21,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
|
|
@ -75,13 +74,11 @@ type indexReader struct {
|
||||||
descList []*indexBlockDesc
|
descList []*indexBlockDesc
|
||||||
readers map[uint32]*blockReader
|
readers map[uint32]*blockReader
|
||||||
state stateIdent
|
state stateIdent
|
||||||
timings *readTimings
|
|
||||||
cacher *historyCacher
|
cacher *historyCacher
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadIndexData loads the index data associated with the specified state.
|
// loadIndexData loads the index data associated with the specified state.
|
||||||
func loadIndexData(db ethdb.KeyValueReader, state stateIdent, timings *readTimings, cacher *historyCacher) ([]*indexBlockDesc, error) {
|
func loadIndexData(db ethdb.KeyValueReader, state stateIdent, cacher *historyCacher) ([]*indexBlockDesc, error) {
|
||||||
start := time.Now()
|
|
||||||
key := state.CacheKey()
|
key := state.CacheKey()
|
||||||
var blob []byte
|
var blob []byte
|
||||||
if cacher != nil && cacher.index.Contains(key) {
|
if cacher != nil && cacher.index.Contains(key) {
|
||||||
|
|
@ -93,9 +90,6 @@ func loadIndexData(db ethdb.KeyValueReader, state stateIdent, timings *readTimin
|
||||||
blob = rawdb.ReadStorageHistoryIndex(db, state.addressHash, state.storageHash)
|
blob = rawdb.ReadStorageHistoryIndex(db, state.addressHash, state.storageHash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if timings != nil {
|
|
||||||
timings.kvdbIndex = time.Since(start)
|
|
||||||
}
|
|
||||||
if len(blob) == 0 {
|
if len(blob) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
@ -108,12 +102,7 @@ func loadIndexData(db ethdb.KeyValueReader, state stateIdent, timings *readTimin
|
||||||
// newIndexReader constructs a index reader for the specified state. Reader with
|
// newIndexReader constructs a index reader for the specified state. Reader with
|
||||||
// empty data is allowed.
|
// empty data is allowed.
|
||||||
func newIndexReader(db ethdb.KeyValueReader, state stateIdent, cacher *historyCacher) (*indexReader, error) {
|
func newIndexReader(db ethdb.KeyValueReader, state stateIdent, cacher *historyCacher) (*indexReader, error) {
|
||||||
return newIndexReaderWithTimings(db, state, nil, cacher)
|
descList, err := loadIndexData(db, state, cacher)
|
||||||
}
|
|
||||||
|
|
||||||
// Helper to allow passing timings
|
|
||||||
func newIndexReaderWithTimings(db ethdb.KeyValueReader, state stateIdent, timings *readTimings, cacher *historyCacher) (*indexReader, error) {
|
|
||||||
descList, err := loadIndexData(db, state, timings, cacher)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -122,7 +111,6 @@ func newIndexReaderWithTimings(db ethdb.KeyValueReader, state stateIdent, timing
|
||||||
readers: make(map[uint32]*blockReader),
|
readers: make(map[uint32]*blockReader),
|
||||||
db: db,
|
db: db,
|
||||||
state: state,
|
state: state,
|
||||||
timings: timings,
|
|
||||||
cacher: cacher,
|
cacher: cacher,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
@ -138,7 +126,7 @@ func (r *indexReader) refresh() error {
|
||||||
delete(r.readers, last.id)
|
delete(r.readers, last.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
descList, err := loadIndexData(r.db, r.state, r.timings, nil)
|
descList, err := loadIndexData(r.db, r.state, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -163,7 +151,6 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) {
|
||||||
err error
|
err error
|
||||||
blob []byte
|
blob []byte
|
||||||
)
|
)
|
||||||
start := time.Now()
|
|
||||||
key := fmt.Sprintf("%s:%d", r.state.CacheKey(), desc.id)
|
key := fmt.Sprintf("%s:%d", r.state.CacheKey(), desc.id)
|
||||||
if r.cacher != nil && r.cacher.block.Contains(key) {
|
if r.cacher != nil && r.cacher.block.Contains(key) {
|
||||||
blob, _ = r.cacher.block.Get(key)
|
blob, _ = r.cacher.block.Get(key)
|
||||||
|
|
@ -178,9 +165,6 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) {
|
||||||
r.cacher.block.Add(key, blob)
|
r.cacher.block.Add(key, blob)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if r.timings != nil {
|
|
||||||
r.timings.kvdbBlock = time.Since(start)
|
|
||||||
}
|
|
||||||
br, err = newBlockReader(blob)
|
br, err = newBlockReader(blob)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
|
|
|
||||||
|
|
@ -208,7 +208,6 @@ func (b *batchIndexer) finish(force bool) error {
|
||||||
if err := eg.Wait(); err != nil {
|
if err := eg.Wait(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
mtime := time.Now()
|
|
||||||
|
|
||||||
// Update the position of last indexed state history
|
// Update the position of last indexed state history
|
||||||
if !b.delete {
|
if !b.delete {
|
||||||
|
|
@ -220,14 +219,10 @@ func (b *batchIndexer) finish(force bool) error {
|
||||||
storeIndexMetadata(batch, b.lastID-1)
|
storeIndexMetadata(batch, b.lastID-1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
wtime := time.Now()
|
|
||||||
if err := batch.Write(); err != nil {
|
if err := batch.Write(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Debug("Committed batch indexer", "accounts", len(b.accounts), "storages", storages, "records", b.counter, "elapsed", common.PrettyDuration(time.Since(start)),
|
log.Debug("Committed batch indexer", "accounts", len(b.accounts), "storages", storages, "records", b.counter, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||||
"mtime", common.PrettyDuration(mtime.Sub(start)),
|
|
||||||
"utime", common.PrettyDuration(wtime.Sub(mtime)),
|
|
||||||
"wtime", common.PrettyDuration(time.Since(wtime)))
|
|
||||||
b.counter = 0
|
b.counter = 0
|
||||||
b.accounts = make(map[common.Hash][]uint64)
|
b.accounts = make(map[common.Hash][]uint64)
|
||||||
b.storages = make(map[common.Hash]map[common.Hash][]uint64)
|
b.storages = make(map[common.Hash]map[common.Hash][]uint64)
|
||||||
|
|
|
||||||
|
|
@ -23,14 +23,12 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/common/lru"
|
"github.com/ethereum/go-ethereum/common/lru"
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// stateIdent represents the identifier of a state element, which can be
|
// stateIdent represents the identifier of a state element, which can be
|
||||||
|
|
@ -134,9 +132,8 @@ type indexReaderWithLimitTag struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// newIndexReaderWithLimitTag constructs a index reader with indexing position.
|
// newIndexReaderWithLimitTag constructs a index reader with indexing position.
|
||||||
// Add timings parameter to pass through
|
func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64, cacher *historyCacher) (*indexReaderWithLimitTag, error) {
|
||||||
func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64, timings *readTimings, cacher *historyCacher) (*indexReaderWithLimitTag, error) {
|
r, err := newIndexReader(db, state, cacher)
|
||||||
r, err := newIndexReaderWithTimings(db, state, timings, cacher)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -226,24 +223,10 @@ func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader, ca
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Struct to collect timing info for account or storage read
|
|
||||||
type readTimings struct {
|
|
||||||
kvdbMeta time.Duration
|
|
||||||
kvdbIndex time.Duration
|
|
||||||
kvdbBlock time.Duration
|
|
||||||
frdbIndex time.Duration
|
|
||||||
frdbHistory time.Duration
|
|
||||||
frdbMeta time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// readAccountMetadata resolves the account metadata within the specified
|
// readAccountMetadata resolves the account metadata within the specified
|
||||||
// state history.
|
// state history.
|
||||||
func (r *historyReader) readAccountMetadata(address common.Address, historyID uint64, timings *readTimings) ([]byte, error) {
|
func (r *historyReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) {
|
||||||
start := time.Now()
|
|
||||||
blob := rawdb.ReadStateAccountIndex(r.freezer, historyID)
|
blob := rawdb.ReadStateAccountIndex(r.freezer, historyID)
|
||||||
if timings != nil {
|
|
||||||
timings.frdbIndex = time.Since(start)
|
|
||||||
}
|
|
||||||
if len(blob) == 0 {
|
if len(blob) == 0 {
|
||||||
return nil, fmt.Errorf("account index is truncated, historyID: %d", historyID)
|
return nil, fmt.Errorf("account index is truncated, historyID: %d", historyID)
|
||||||
}
|
}
|
||||||
|
|
@ -268,13 +251,9 @@ func (r *historyReader) readAccountMetadata(address common.Address, historyID ui
|
||||||
|
|
||||||
// readStorageMetadata resolves the storage slot metadata within the specified
|
// readStorageMetadata resolves the storage slot metadata within the specified
|
||||||
// state history.
|
// state history.
|
||||||
func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int, timings *readTimings) ([]byte, error) {
|
func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) {
|
||||||
// TODO(rj493456442) optimize it with partial read
|
// TODO(rj493456442) optimize it with partial read
|
||||||
start := time.Now()
|
|
||||||
blob := rawdb.ReadStateStorageIndex(r.freezer, historyID)
|
blob := rawdb.ReadStateStorageIndex(r.freezer, historyID)
|
||||||
if timings != nil {
|
|
||||||
timings.frdbIndex = time.Since(start)
|
|
||||||
}
|
|
||||||
if len(blob) == 0 {
|
if len(blob) == 0 {
|
||||||
return nil, fmt.Errorf("storage index is truncated, historyID: %d", historyID)
|
return nil, fmt.Errorf("storage index is truncated, historyID: %d", historyID)
|
||||||
}
|
}
|
||||||
|
|
@ -291,11 +270,7 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash
|
||||||
m meta
|
m meta
|
||||||
target common.Hash
|
target common.Hash
|
||||||
)
|
)
|
||||||
startMeta := time.Now()
|
|
||||||
blob = rawdb.ReadStateHistoryMeta(r.freezer, historyID)
|
blob = rawdb.ReadStateHistoryMeta(r.freezer, historyID)
|
||||||
if timings != nil {
|
|
||||||
timings.frdbMeta = time.Since(startMeta)
|
|
||||||
}
|
|
||||||
if err := m.decode(blob); err != nil {
|
if err := m.decode(blob); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -319,8 +294,8 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash
|
||||||
}
|
}
|
||||||
|
|
||||||
// readAccount retrieves the account data from the specified state history.
|
// readAccount retrieves the account data from the specified state history.
|
||||||
func (r *historyReader) readAccount(address common.Address, historyID uint64, timings *readTimings) ([]byte, error) {
|
func (r *historyReader) readAccount(address common.Address, historyID uint64) ([]byte, error) {
|
||||||
metadata, err := r.readAccountMetadata(address, historyID, timings)
|
metadata, err := r.readAccountMetadata(address, historyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -328,11 +303,7 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64, ti
|
||||||
offset := int(binary.BigEndian.Uint32(metadata[common.AddressLength+1 : common.AddressLength+5])) // four bytes for the account data offset
|
offset := int(binary.BigEndian.Uint32(metadata[common.AddressLength+1 : common.AddressLength+5])) // four bytes for the account data offset
|
||||||
|
|
||||||
// TODO(rj493456442) optimize it with partial read
|
// TODO(rj493456442) optimize it with partial read
|
||||||
start := time.Now()
|
|
||||||
data := rawdb.ReadStateAccountHistory(r.freezer, historyID)
|
data := rawdb.ReadStateAccountHistory(r.freezer, historyID)
|
||||||
if timings != nil {
|
|
||||||
timings.frdbHistory = time.Since(start)
|
|
||||||
}
|
|
||||||
if len(data) < length+offset {
|
if len(data) < length+offset {
|
||||||
return nil, fmt.Errorf("account data is truncated, address: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, historyID, len(data), offset, length)
|
return nil, fmt.Errorf("account data is truncated, address: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, historyID, len(data), offset, length)
|
||||||
}
|
}
|
||||||
|
|
@ -340,8 +311,8 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64, ti
|
||||||
}
|
}
|
||||||
|
|
||||||
// readStorage retrieves the storage slot data from the specified state history.
|
// readStorage retrieves the storage slot data from the specified state history.
|
||||||
func (r *historyReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64, timings *readTimings) ([]byte, error) {
|
func (r *historyReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) {
|
||||||
metadata, err := r.readAccountMetadata(address, historyID, nil) // No timings for account metadata in storage read
|
metadata, err := r.readAccountMetadata(address, historyID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -352,7 +323,7 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha
|
||||||
slotIndexOffset := int(binary.BigEndian.Uint32(metadata[common.AddressLength+5 : common.AddressLength+9]))
|
slotIndexOffset := int(binary.BigEndian.Uint32(metadata[common.AddressLength+5 : common.AddressLength+9]))
|
||||||
slotIndexNumber := int(binary.BigEndian.Uint32(metadata[common.AddressLength+9 : common.AddressLength+13]))
|
slotIndexNumber := int(binary.BigEndian.Uint32(metadata[common.AddressLength+9 : common.AddressLength+13]))
|
||||||
|
|
||||||
slotMetadata, err := r.readStorageMetadata(storageKey, storageHash, historyID, slotIndexOffset, slotIndexNumber, timings)
|
slotMetadata, err := r.readStorageMetadata(storageKey, storageHash, historyID, slotIndexOffset, slotIndexNumber)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -360,11 +331,7 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha
|
||||||
offset := int(binary.BigEndian.Uint32(slotMetadata[common.HashLength+1 : common.HashLength+5])) // four bytes for slot data offset
|
offset := int(binary.BigEndian.Uint32(slotMetadata[common.HashLength+1 : common.HashLength+5])) // four bytes for slot data offset
|
||||||
|
|
||||||
// TODO(rj493456442) optimize it with partial read
|
// TODO(rj493456442) optimize it with partial read
|
||||||
start := time.Now()
|
|
||||||
data := rawdb.ReadStateStorageHistory(r.freezer, historyID)
|
data := rawdb.ReadStateStorageHistory(r.freezer, historyID)
|
||||||
if timings != nil {
|
|
||||||
timings.frdbHistory = time.Since(start)
|
|
||||||
}
|
|
||||||
if len(data) < offset+length {
|
if len(data) < offset+length {
|
||||||
return nil, fmt.Errorf("storage data is truncated, address: %#x, key: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, storageKey, historyID, len(data), offset, length)
|
return nil, fmt.Errorf("storage data is truncated, address: %#x, key: %#x, historyID: %d, size: %d, offset: %d, len: %d", address, storageKey, historyID, len(data), offset, length)
|
||||||
}
|
}
|
||||||
|
|
@ -376,8 +343,6 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha
|
||||||
// lastID: represents the ID of the latest/newest state history;
|
// lastID: represents the ID of the latest/newest state history;
|
||||||
// latestValue: represents the state value at the current disk layer with ID == lastID;
|
// latestValue: represents the state value at the current disk layer with ID == lastID;
|
||||||
func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) {
|
func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) {
|
||||||
origin := time.Now()
|
|
||||||
timings := &readTimings{}
|
|
||||||
tail, err := r.freezer.Tail()
|
tail, err := r.freezer.Tail()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -391,9 +356,7 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6
|
||||||
// To serve the request, all state histories from stateID+1 to lastID
|
// To serve the request, all state histories from stateID+1 to lastID
|
||||||
// must be indexed. It's not supposed to happen unless system is very
|
// must be indexed. It's not supposed to happen unless system is very
|
||||||
// wrong.
|
// wrong.
|
||||||
start := time.Now()
|
|
||||||
metadata := loadIndexMetadata(r.disk)
|
metadata := loadIndexMetadata(r.disk)
|
||||||
timings.kvdbMeta = time.Since(start)
|
|
||||||
if metadata == nil || metadata.Last < lastID {
|
if metadata == nil || metadata.Last < lastID {
|
||||||
indexed := "null"
|
indexed := "null"
|
||||||
if metadata != nil {
|
if metadata != nil {
|
||||||
|
|
@ -406,13 +369,12 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6
|
||||||
// state retrieval
|
// state retrieval
|
||||||
ir, ok := r.readers[state.String()]
|
ir, ok := r.readers[state.String()]
|
||||||
if !ok {
|
if !ok {
|
||||||
ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, metadata.Last, timings, r.cacher)
|
ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, metadata.Last, r.cacher)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
r.readers[state.String()] = ir
|
r.readers[state.String()] = ir
|
||||||
}
|
}
|
||||||
ir.reader.timings = timings
|
|
||||||
historyID, err := ir.readGreaterThan(stateID, lastID)
|
historyID, err := ir.readGreaterThan(stateID, lastID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
@ -428,17 +390,10 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6
|
||||||
// Such truncation should be captured by the state resolver below, rather than returning
|
// Such truncation should be captured by the state resolver below, rather than returning
|
||||||
// invalid data.
|
// invalid data.
|
||||||
var result []byte
|
var result []byte
|
||||||
var item string
|
|
||||||
if state.account {
|
if state.account {
|
||||||
item = "account"
|
result, err = r.readAccount(state.address, historyID)
|
||||||
result, err = r.readAccount(state.address, historyID, timings)
|
|
||||||
} else {
|
} else {
|
||||||
item = "storage"
|
result, err = r.readStorage(state.address, state.storageKey, state.storageHash, historyID)
|
||||||
result, err = r.readStorage(state.address, state.storageKey, state.storageHash, historyID, timings)
|
|
||||||
}
|
}
|
||||||
log.Info("HistoryRead", "item", item, "elapsed", time.Since(origin),
|
|
||||||
"kvdbMeta", timings.kvdbMeta, "kvdbIndex", timings.kvdbIndex, "kvdbBlock", timings.kvdbBlock,
|
|
||||||
"frdbIndex", timings.frdbIndex, "frdbHistory", timings.frdbHistory, "frdbMeta", timings.frdbMeta,
|
|
||||||
)
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue