forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 14:46:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

cmd/geth, core/state, trie: don't iterate known subtries

Browse source
This commit is contained in:
Péter Szilágyi 2016-10-25 19:37:15 +03:00
parent 2f913fbc01
commit cb3e606012
No known key found for this signature in database
GPG key ID: 119A76381CCB7DD2
5 changed files with 74 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/geth/chaincmd.go
View file

@ -119,7 +119,10 @@ func importChain(ctx *cli.Context) error {
}
fmt.Println(stats)
fmt.Printf("Trie cache misses: %d\n", trie.CacheMisses())
fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
fmt.Printf("Trie cache unloads: %d\n", trie.CacheUnloads())
fmt.Printf("Direct cache reads: %d\n", trie.DirectCacheReads())
fmt.Printf("Direct cache writes: %d\n", trie.DirectCacheWrites())
fmt.Printf("Direct cache misses: %d\n\n", trie.DirectCacheMisses())
// Print the memory statistics used by the importing
mem := new(runtime.MemStats)
@ -308,7 +311,7 @@ func buildCache(ctx *cli.Context) error {
}
i += 1
if i % 10000 == 0 {
if i%10000 == 0 {
fmt.Printf("Processed %d accounts, at %v\n", i, common.ToHex(iter.Key))
}
}

4
core/state/sync.go
View file

@ -53,12 +53,12 @@ func NewStateSync(number uint64, hash common.Hash, root common.Hash, database et
}
}
// Schedule downloading all the dependencies of the account
syncer.AddSubTrie(obj.Root, 64, parent, nil)
syncer.AddSubTrie(obj.Root, 64, parent, nil, nil)
syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent)
return nil
}
syncer = trie.NewTrieSync(root, database, callback)
syncer = trie.NewTrieSync(root, database, callback, CachePrefix)
return (*StateSync)(syncer)
}

38
trie/directcache.go
View file

@ -30,6 +30,27 @@ var directCacheWrites = metrics.NewCounter("directcache/writes")
var directCacheHitTimer = metrics.NewTimer("directcache/timer/hits")
var directCacheMissTimer = metrics.NewTimer("directcache/timer/misses")
// DirectCacheReads retrieves a global counter measuring the number of direct
// cache reads from the disk since process startup. This isn't useful for anything
// apart from trie debugging purposes.
func DirectCacheReads() int64 {
return directCacheHitTimer.Count() + directCacheMissTimer.Count()
}
// DirectCacheWrites retrieves a global counter measuring the number of direct
// cache writes from the disk since process startup. This isn't useful for anything
// apart from trie debugging purposes.
func DirectCacheWrites() int64 {
return directCacheWrites.Count()
}
// DirectCacheMisses retrieves a global counter measuring the number of direct
// cache writes from the disk since process startup. This isn't useful for anything
// apart from trie debugging purposes.
func DirectCacheMisses() int64 {
return directCacheMissTimer.Count()
}
type cachedValue struct {
Value []byte
BlockNum uint64
@ -157,7 +178,6 @@ func (dc *DirectCache) TryDelete(key []byte) error {
}
func (dc *DirectCache) CommitTo(dbw DatabaseWriter) (root common.Hash, err error) {
directCacheWrites.Inc(int64(len(dc.dirty)))
for k, _ := range dc.dirty {
v, err := dc.data.TryGet([]byte(k))
if err, ok := err.(*MissingNodeError); err != nil && !ok {
@ -175,7 +195,23 @@ func (dc *DirectCache) putCache(dbw DatabaseWriter, key, value []byte) error {
return WriteDirectCache(dc.keyPrefix, key, value, dc.blockNum, dc.blockHash, dbw)
}
// WriteDirectCache places a value node directly into the database along with
// block metadata to validate its relevancy.
//
// The method is meant to be used by code that circumvents the state database
// and its integrated cache, namely during fast sync and database upgrades.
func WriteDirectCache(prefix, key, value []byte, number uint64, hash common.Hash, dbw DatabaseWriter) error {
directCacheWrites.Inc(1)
enc, _ := rlp.EncodeToBytes(cachedValue{value, number, hash})
return dbw.Put(append(prefix, key...), enc)
}
// GetDirectCache retrieves a value node directly from the database along with
// block metadata to validate its relevancy.
//
// The method is meant to be used by code that circumvents the state database
// and its integrated cache, namely during fast sync and database upgrades.
func GetDirectCache(prefix, key []byte, db Database) ([]byte, error) {
defer func(start time.Time) { directCacheHitTimer.UpdateSince(start) }(time.Now())
return db.Get(append(prefix, key...))
}

26
trie/sync.go
View file

@ -41,6 +41,7 @@ type request struct {
deps int // Number of dependencies before allowed to commit this node
callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
dcPrefix []byte // Database prefix to use for leaf direct caching
}
// keys traverses the request ancestry tree, reconstructing the key paths leading
@ -92,18 +93,18 @@ type TrieSync struct {
}
// NewTrieSync creates a new trie data download scheduler.
func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback, dcPrefix []byte) *TrieSync {
ts := &TrieSync{
database: database,
requests: make(map[common.Hash]*request),
queue: prque.New(),
}
ts.AddSubTrie(root, 0, common.Hash{}, callback)
ts.AddSubTrie(root, 0, common.Hash{}, callback, dcPrefix)
return ts
}
// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback, dcPrefix []byte) {
// Short circuit if the trie is empty or already known
if root == emptyRoot {
return
@ -118,6 +119,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
hash: root,
depth: depth,
callback: callback,
dcPrefix: dcPrefix,
}
// If this sub-trie has a designated parent, link them together
if parent != (common.Hash{}) {
@ -280,13 +282,26 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
blob, _ := s.database.Get(node)
if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil {
// Local node found, iterate and invoke the leaf callback for all subleaves
if req.callback != nil {
if req.callback != nil && req.dcPrefix != nil {
trie, _ := New(common.BytesToHash(node[:]), s.database, 0)
unknown := false
it := NewNodeIterator(trie)
for it.Next() {
if it.Leaf {
req.callback(req.keys(append(child.path, it.Nibbles...)), nil, req.hash)
// If this branch was already visited, abort iteration
keys := req.keys(append(child.path, it.Nibbles...))
if !unknown {
if val, err := GetDirectCache(req.dcPrefix, keys[0], s.database); val != nil && err == nil {
break
}
// Branch is unknown, mark as so to avoid repeated db lookups
unknown = true
}
// Otherwise write out all accounts ending in this leaf
if err := req.callback(keys, nil, req.hash); err != nil {
return nil, err
}
}
}
}
@ -299,6 +314,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
nibbles: [][]byte{child.path},
depth: child.depth,
callback: req.callback,
dcPrefix: req.dcPrefix,
})
}
}

16
trie/sync_test.go
View file

@ -115,7 +115,7 @@ func TestEmptyTrieSync(t *testing.T) {
for i, trie := range []*Trie{emptyA, emptyB} {
db, _ := ethdb.NewMemDatabase()
if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil, nil).Missing(1); len(req) != 0 {
t.Errorf("test %d: content requested for empty trie: %v", i, req)
}
}
@ -132,7 +132,7 @@ func testIterativeTrieSync(t *testing.T, batch int) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
queue := append([]common.Hash{}, sched.Missing(batch)...)
for len(queue) > 0 {
@ -161,7 +161,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
queue := append([]common.Hash{}, sched.Missing(10000)...)
for len(queue) > 0 {
@ -195,7 +195,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(batch) {
@ -232,7 +232,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
queue := make(map[common.Hash]struct{})
for _, hash := range sched.Missing(10000) {
@ -275,7 +275,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
queue := append([]common.Hash{}, sched.Missing(0)...)
requested := make(map[common.Hash]struct{})
@ -311,7 +311,7 @@ func TestIncompleteTrieSync(t *testing.T) {
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
added := []common.Hash{}
queue := append([]common.Hash{}, sched.Missing(1)...)
@ -371,7 +371,7 @@ func TestAllLeafCallbackTrieSync(t *testing.T) {
}
// Create a destination trie and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, callback)
sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, callback, []byte("leaf:"))
queue := append([]common.Hash{}, sched.Missing(1)...)
for len(queue) > 0 {