From 0d93fc76b2e6fa4be18b6b92e3dc96e8ab519f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 22 Jun 2018 15:37:43 +0300 Subject: [PATCH] core, ethdb, trie: historical state pruning --- core/blockchain.go | 10 +- core/genesis.go | 1 + core/state/database.go | 4 +- core/state/statedb.go | 11 +- core/state/sync.go | 2 +- eth/api_tracer.go | 14 +- les/handler.go | 2 +- trie/database.go | 561 ++++++++++++++++++++++++++++++----------- trie/hasher.go | 25 +- trie/iterator.go | 7 +- trie/proof.go | 6 +- trie/secure_trie.go | 20 +- trie/sync.go | 2 +- trie/trie.go | 28 +- 14 files changed, 495 insertions(+), 198 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index b6605e66c9..b4f39d87fe 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -684,7 +684,7 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types. // TrieNode retrieves a blob of data associated with a trie node (or code hash) // either from ephemeral in-memory cache, or from persistent storage. func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { - return bc.stateCache.TrieDB().Node(hash) + return bc.stateCache.TrieDB().Node(common.Hash{}, hash) // TODO(karalabe): make this work again } // Stop stops the blockchain service. If any imports are currently in progress @@ -719,10 +719,10 @@ func (bc *BlockChain) Stop() { } } for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + triedb.Dereference(bc.triegc.PopItem().(common.Hash), false) } if size, _ := triedb.Size(); size != 0 { - log.Error("Dangling trie nodes after full cleanup") + log.Error("Dangling trie nodes after full cleanup", "size", size) } } log.Info("Blockchain manager stopped") @@ -966,7 +966,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } } else { // Full but not archive node, do proper garbage collection - triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive + triedb.Reference(common.Hash{}, root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) if current := block.NumberU64(); current > triesInMemory { @@ -1007,7 +1007,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.triegc.Push(root, number) break } - triedb.Dereference(root.(common.Hash)) + triedb.Dereference(root.(common.Hash), true) } } } diff --git a/core/genesis.go b/core/genesis.go index cbb6eecd28..ce5bf6eb9e 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -260,6 +260,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { head.Difficulty = params.GenesisDifficulty } statedb.Commit(false) + statedb.Database().TrieDB().Reference(common.Hash{}, root, common.Hash{}) statedb.Database().TrieDB().Commit(root, true) return types.NewBlock(head, nil, nil, nil) diff --git a/core/state/database.go b/core/state/database.go index f6ea144b9b..01872bf3c4 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -127,7 +127,7 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - return trie.NewSecure(root, db.db, 0) + return trie.NewSecureWithOwner(addrHash, root, db.db, 0) } // CopyTrie returns an independent copy of the given trie. @@ -144,7 +144,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - code, err := db.db.Node(codeHash) + code, err := db.db.Node(common.Hash{}, codeHash) if err == nil { db.codeSizeCache.Add(codeHash, len(code)) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 8ad25a5824..852eab8f53 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -635,7 +636,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) case isDirty: // Write any contract code associated with the state object if stateObject.code != nil && stateObject.dirtyCode { - s.db.TrieDB().InsertBlob(common.BytesToHash(stateObject.CodeHash()), stateObject.code) + s.db.TrieDB().DiskDB().(ethdb.Database).Put(stateObject.CodeHash(), stateObject.code) stateObject.dirtyCode = false } // Write any storage changes in the state object to its storage trie. @@ -648,17 +649,13 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) delete(s.stateObjectsDirty, addr) } // Write trie changes. - root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error { + root, err = s.trie.Commit(func(owner common.Hash, leaf []byte, parent common.Hash) error { var account Account if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } if account.Root != emptyState { - s.db.TrieDB().Reference(account.Root, parent) - } - code := common.BytesToHash(account.CodeHash) - if code != emptyCode { - s.db.TrieDB().Reference(code, parent) + s.db.TrieDB().Reference(owner, account.Root, parent) } return nil }) diff --git a/core/state/sync.go b/core/state/sync.go index c566e79073..1689a0ce73 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -27,7 +27,7 @@ import ( // NewStateSync create a new state trie download scheduler. func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.Sync { var syncer *trie.Sync - callback := func(leaf []byte, parent common.Hash) error { + callback := func(owner common.Hash, leaf []byte, parent common.Hash) error { var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err diff --git a/eth/api_tracer.go b/eth/api_tracer.go index a529ea118e..b682376ec5 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -304,14 +304,14 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl failed = err break } - // Reference the trie twice, once for us, once for the tracer - database.TrieDB().Reference(root, common.Hash{}) + // Reference the trie twice, once for us, once for the trancer + database.TrieDB().Reference(common.Hash{}, root, common.Hash{}) if number >= origin { - database.TrieDB().Reference(root, common.Hash{}) + database.TrieDB().Reference(common.Hash{}, root, common.Hash{}) } // Dereference all past tries we ourselves are done working with if proot != (common.Hash{}) { - database.TrieDB().Dereference(proot) + database.TrieDB().Dereference(proot, false) } proot = root @@ -335,7 +335,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl done[uint64(result.Block)] = result // Dereference any paret tries held in memory by this task - database.TrieDB().Dereference(res.rootref) + database.TrieDB().Dereference(res.rootref, false) // Stream completed traces to the user, aborting on the first error for result, ok := done[next]; ok; result, ok = done[next] { @@ -688,9 +688,9 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* if err := statedb.Reset(root); err != nil { return nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) } - database.TrieDB().Reference(root, common.Hash{}) + database.TrieDB().Reference(common.Hash{}, root, common.Hash{}) if proot != (common.Hash{}) { - database.TrieDB().Dereference(proot) + database.TrieDB().Dereference(proot, false) } proot = root } diff --git a/les/handler.go b/les/handler.go index 46a1ed2d7b..eb89d032f4 100644 --- a/les/handler.go +++ b/les/handler.go @@ -631,7 +631,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err != nil { continue } - code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) + code, _ := statedb.Database().TrieDB().Node(common.Hash{}, common.BytesToHash(account.CodeHash)) // TODO(karalabe): make this work again data = append(data, code) if bytes += len(code); bytes >= softResponseLimit { diff --git a/trie/database.go b/trie/database.go index 958823eb8d..7eb501b860 100644 --- a/trie/database.go +++ b/trie/database.go @@ -19,6 +19,7 @@ package trie import ( "fmt" "io" + "math/big" "sync" "time" @@ -44,6 +45,10 @@ var ( memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) + memcachePruneTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/prune/time", nil) + memcachePruneNodesMeter = metrics.NewRegisteredMeter("trie/memcache/prune/nodes", nil) + memcachePruneSizeMeter = metrics.NewRegisteredMeter("trie/memcache/prune/size", nil) + memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) @@ -52,8 +57,9 @@ var ( // secureKeyPrefix is the database key prefix used to store trie node preimages. var secureKeyPrefix = []byte("secure-key-") -// secureKeyLength is the length of the above prefix + 32byte hash. -const secureKeyLength = 11 + 32 +// metaRoot is the identifier of the global memcache root that anchors the block +// accounts tries for garbage collection. +const metaRoot = "" // DatabaseReader wraps the Get and Has method of a backing store for the trie. type DatabaseReader interface { @@ -64,24 +70,55 @@ type DatabaseReader interface { Has(key []byte) (bool, error) } +// makeNodeKey returns the database key for a trie node. +func makeNodeKey(owner common.Hash, hash common.Hash) string { + if hash == (common.Hash{}) { + return metaRoot + } + if owner == (common.Hash{}) { + return string(hash[:]) + } + return string(append(owner[:], hash[:]...)) +} + +// splitNodeKey returns the composing hashes of a trie node key. +func splitNodeKey(key string) (common.Hash, common.Hash) { + switch len(key) { + case 0: + return common.Hash{}, common.Hash{} + + case common.HashLength: + return common.Hash{}, common.BytesToHash([]byte(key)) + + case 2 * common.HashLength: + return common.BytesToHash([]byte(key[:common.HashLength])), common.BytesToHash([]byte(key[common.HashLength:])) + + default: + panic(fmt.Sprintf("invalid node key: %s", key)) + } +} + // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes - cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs - dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes - oldest common.Hash // Oldest tracked node, flush-list head - newest common.Hash // Newest tracked node, flush-list tail + cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs + dirties map[string]*cachedNode // Data and references relationships of dirty nodes + oldest string // Oldest tracked node, flush-list head + newest string // Newest tracked node, flush-list tail preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit + prunetime time.Duration // Time spend on disk pruning since last commit + prunenodes uint64 // Nodes pruned from disk since last commit + prunesize common.StorageSize // Data storage pruned from disk since last commit + flushtime time.Duration // Time spent on data flushing since last commit flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit @@ -108,7 +145,6 @@ type rawFullNode [17]node func (n rawFullNode) canUnload(uint16, uint16) bool { panic("this should never end up in a live trie") } func (n rawFullNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") } -func (n rawFullNode) fstring(ind string) string { panic("this should never end up in a live trie") } func (n rawFullNode) EncodeRLP(w io.Writer) error { var nodes [17]node @@ -123,6 +159,20 @@ func (n rawFullNode) EncodeRLP(w io.Writer) error { return rlp.Encode(w, nodes) } +func (n rawFullNode) String() string { return n.fstring("") } + +func (n rawFullNode) fstring(ind string) string { + resp := fmt.Sprintf("[\n%s ", ind) + for i, node := range n { + if node == nil { + resp += fmt.Sprintf("%s: ", indices[i]) + } else { + resp += fmt.Sprintf("%s: %v", indices[i], node.fstring(ind+" ")) + } + } + return resp + fmt.Sprintf("\n%s] ", ind) +} + // rawShortNode represents only the useful data content of a short node, with the // caches and flags stripped out to minimize its data storage. This type honors // the same RLP encoding as the original parent. @@ -131,9 +181,20 @@ type rawShortNode struct { Val node } -func (n rawShortNode) canUnload(uint16, uint16) bool { panic("this should never end up in a live trie") } -func (n rawShortNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") } -func (n rawShortNode) fstring(ind string) string { panic("this should never end up in a live trie") } +func (n *rawShortNode) canUnload(uint16, uint16) bool { + panic("this should never end up in a live trie") +} +func (n *rawShortNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") } + +func (n *rawShortNode) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, &shortNode{Key: hexToCompact(n.Key), Val: n.Val}) +} + +func (n *rawShortNode) String() string { return n.fstring("") } + +func (n *rawShortNode) fstring(ind string) string { + return fmt.Sprintf("{%x: %v} ", n.Key, n.Val.fstring(ind+" ")) +} // cachedNode is all the information we know about a single cached node in the // memory database write layer. @@ -141,11 +202,11 @@ type cachedNode struct { node node // Cached collapsed trie node, or raw rlp data size uint16 // Byte size of the useful cached data - parents uint32 // Number of live nodes referencing this one - children map[common.Hash]uint16 // External children referenced by this node + parents uint32 // Number of live nodes referencing this one + children map[string]uint16 // External children referenced by this node - flushPrev common.Hash // Previous node in the flush-list - flushNext common.Hash // Next node in the flush-list + flushPrev string // Previous node in the flush-list + flushNext string // Next node in the flush-list } // rlp returns the raw rlp encoded blob of the cached node, either directly from @@ -170,34 +231,46 @@ func (n *cachedNode) obj(hash common.Hash, cachegen uint16) node { return expandNode(hash[:], n.node, cachegen) } -// childs returns all the tracked children of this node, both the implicit ones -// from inside the node as well as the explicit ones from outside the node. -func (n *cachedNode) childs() []common.Hash { - children := make([]common.Hash, 0, 16) - for child := range n.children { - children = append(children, child) +// iterateRefs walks the embedded children of the cached node, tracking the +// internal path and invoking the provided callback on all hash nodes. +func (n *cachedNode) iterateRefs(path []byte, onHashNode func([]byte, common.Hash) error) error { + if _, ok := n.node.(rawNode); ok { + return nil } - if _, ok := n.node.(rawNode); !ok { - gatherChildren(n.node, &children) - } - return children + return iterateRefs(n.node, path, onHashNode) } -// gatherChildren traverses the node hierarchy of a collapsed storage node and -// retrieves all the hashnode children. -func gatherChildren(n node, children *[]common.Hash) { +// iterateRefs traverses the node hierarchy of a cached node and invokes the +// provided callback on all hash nodes. +func iterateRefs(n node, path []byte, onHashNode func([]byte, common.Hash) error) error { switch n := n.(type) { case *rawShortNode: - gatherChildren(n.Val, children) + return iterateRefs(n.Val, append(path, n.Key...), onHashNode) + + case *shortNode: + return iterateRefs(n.Val, append(path, n.Key...), onHashNode) case rawFullNode: for i := 0; i < 16; i++ { - gatherChildren(n[i], children) + if err := iterateRefs(n[i], append(path, byte(i)), onHashNode); err != nil { + return err + } } + return nil + + case *fullNode: + for i := 0; i < 16; i++ { + if err := iterateRefs(n.Children[i], append(path, byte(i)), onHashNode); err != nil { + return err + } + } + return nil + case hashNode: - *children = append(*children, common.BytesToHash(n)) + return onHashNode(path, common.BytesToHash(n)) case valueNode, nil: + return nil default: panic(fmt.Sprintf("unknown node type: %T", n)) @@ -210,7 +283,7 @@ func simplifyNode(n node) node { switch n := n.(type) { case *shortNode: // Short nodes discard the flags and cascade - return &rawShortNode{Key: n.Key, Val: simplifyNode(n.Val)} + return &rawShortNode{Key: compactToHex(n.Key), Val: simplifyNode(n.Val)} case *fullNode: // Full nodes discard the flags and cascade @@ -237,7 +310,7 @@ func expandNode(hash hashNode, n node, cachegen uint16) node { case *rawShortNode: // Short nodes need key and child expansion return &shortNode{ - Key: compactToHex(n.Key), + Key: n.Key, Val: expandNode(nil, n.Val, cachegen), flags: nodeFlag{ hash: hash, @@ -292,7 +365,7 @@ func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database { return &Database{ diskdb: diskdb, cleans: cleans, - dirties: map[common.Hash]*cachedNode{{}: {}}, + dirties: map[string]*cachedNode{metaRoot: {}}, preimages: make(map[common.Hash][]byte), } } @@ -306,20 +379,22 @@ func (db *Database) DiskDB() DatabaseReader { // yet unknown. This method should only be used for non-trie nodes that require // reference counting, since trie nodes are garbage collected directly through // their embedded children. -func (db *Database) InsertBlob(hash common.Hash, blob []byte) { +func (db *Database) InsertBlob(owner common.Hash, hash common.Hash, blob []byte) { db.lock.Lock() defer db.lock.Unlock() - db.insert(hash, blob, rawNode(blob)) + db.DiskDB().(ethdb.Database).Put([]byte(makeNodeKey(owner, hash)), blob) + //db.insert(owner, hash, blob, rawNode(blob)) } // insert inserts a collapsed trie node into the memory database. This method is // a more generic version of InsertBlob, supporting both raw blob insertions as // well ex trie node insertions. The blob must always be specified to allow proper // size tracking. -func (db *Database) insert(hash common.Hash, blob []byte, node node) { +func (db *Database) insert(owner common.Hash, hash common.Hash, blob []byte, node node) { // If the node's already cached, skip - if _, ok := db.dirties[hash]; ok { + key := makeNodeKey(owner, hash) + if _, ok := db.dirties[key]; ok { return } // Create the cached entry for this node @@ -328,18 +403,20 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { size: uint16(len(blob)), flushPrev: db.newest, } - for _, child := range entry.childs() { - if c := db.dirties[child]; c != nil { + // Track all the implicit references (explicits must be empty) + entry.iterateRefs(nil, func(path []byte, child common.Hash) error { + if c := db.dirties[makeNodeKey(owner, child)]; c != nil { c.parents++ } - } - db.dirties[hash] = entry + return nil + }) + db.dirties[key] = entry // Update the flush-list endpoints - if db.oldest == (common.Hash{}) { - db.oldest, db.newest = hash, hash + if db.oldest == metaRoot { + db.oldest, db.newest = key, key } else { - db.dirties[db.newest].flushNext, db.newest = hash, hash + db.dirties[db.newest].flushNext, db.newest = key, key } db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } @@ -358,10 +435,12 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. -func (db *Database) node(hash common.Hash, cachegen uint16) node { +func (db *Database) node(owner common.Hash, hash common.Hash, cachegen uint16) node { + key := makeNodeKey(owner, hash) + // Retrieve the node from the clean cache if available if db.cleans != nil { - if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + if enc, err := db.cleans.Get(key); err == nil && enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) return mustDecodeNode(hash[:], enc, cachegen) @@ -369,14 +448,14 @@ func (db *Database) node(hash common.Hash, cachegen uint16) node { } // Retrieve the node from the dirty cache if available db.lock.RLock() - dirty := db.dirties[hash] + dirty := db.dirties[key] db.lock.RUnlock() if dirty != nil { return dirty.obj(hash, cachegen) } // Content unavailable in memory, attempt to retrieve from disk - enc, err := db.diskdb.Get(hash[:]) + enc, err := db.diskdb.Get([]byte(key)) if err != nil || enc == nil { return nil } @@ -390,10 +469,12 @@ func (db *Database) node(hash common.Hash, cachegen uint16) node { // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. -func (db *Database) Node(hash common.Hash) ([]byte, error) { +func (db *Database) Node(owner common.Hash, hash common.Hash) ([]byte, error) { + key := makeNodeKey(owner, hash) + // Retrieve the node from the clean cache if available if db.cleans != nil { - if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil { + if enc, err := db.cleans.Get(key); err == nil && enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) return enc, nil @@ -401,17 +482,17 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { } // Retrieve the node from the dirty cache if available db.lock.RLock() - dirty := db.dirties[hash] + dirty := db.dirties[key] db.lock.RUnlock() if dirty != nil { return dirty.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - enc, err := db.diskdb.Get(hash[:]) + enc, err := db.diskdb.Get([]byte(key)) if err == nil && enc != nil { if db.cleans != nil { - db.cleans.Set(string(hash[:]), enc) + db.cleans.Set(key, enc) memcacheCleanMissMeter.Mark(1) memcacheCleanWriteMeter.Mark(int64(len(enc))) } @@ -431,72 +512,69 @@ func (db *Database) preimage(hash common.Hash) ([]byte, error) { return preimage, nil } // Content unavailable in memory, attempt to retrieve from disk - return db.diskdb.Get(db.secureKey(hash[:])) + return db.diskdb.Get(db.preimageKey(hash[:])) } -// secureKey returns the database key for the preimage of key, as an ephemeral -// buffer. The caller must not hold onto the return value because it will become -// invalid on the next call. -func (db *Database) secureKey(key []byte) []byte { - buf := append(db.seckeybuf[:0], secureKeyPrefix...) - buf = append(buf, key...) - return buf +// preimageKey returns the database key for the preimage of key. +func (db *Database) preimageKey(key []byte) []byte { + return append(secureKeyPrefix, key...) } // Nodes retrieves the hashes of all the nodes cached within the memory database. // This method is extremely expensive and should only be used to validate internal // states in test code. -func (db *Database) Nodes() []common.Hash { +func (db *Database) Nodes() []string { db.lock.RLock() defer db.lock.RUnlock() - var hashes = make([]common.Hash, 0, len(db.dirties)) - for hash := range db.dirties { - if hash != (common.Hash{}) { // Special case for "root" references/nodes - hashes = append(hashes, hash) + var keys = make([]string, 0, len(db.dirties)) + for key := range db.dirties { + if key != metaRoot { // Special case for "root" references/nodes + keys = append(keys, key) } } - return hashes + return keys } -// Reference adds a new reference from a parent node to a child node. -func (db *Database) Reference(child common.Hash, parent common.Hash) { +// Reference adds a new reference from a parent node to a child node. We're going +// to break genericity here and assume that parent nodes are not owned (account +// trie) whereas child nodes may be owned (storage trie or bytecode). +func (db *Database) Reference(owner common.Hash, child common.Hash, parent common.Hash) { db.lock.RLock() defer db.lock.RUnlock() - db.reference(child, parent) -} - -// reference is the private locked version of Reference. -func (db *Database) reference(child common.Hash, parent common.Hash) { // If the node does not exist, it's a node pulled from disk, skip - node, ok := db.dirties[child] + childKey := makeNodeKey(owner, child) + node, ok := db.dirties[childKey] if !ok { return } // If the reference already exists, only duplicate for roots - if db.dirties[parent].children == nil { - db.dirties[parent].children = make(map[common.Hash]uint16) - } else if _, ok = db.dirties[parent].children[child]; ok && parent != (common.Hash{}) { + parentKey := makeNodeKey(common.Hash{}, parent) + if db.dirties[parentKey].children == nil { + db.dirties[parentKey].children = make(map[string]uint16) + } else if _, ok = db.dirties[parentKey].children[childKey]; ok && parent != (common.Hash{}) { return } node.parents++ - db.dirties[parent].children[child]++ + db.dirties[parentKey].children[childKey]++ } // Dereference removes an existing reference from a root node. -func (db *Database) Dereference(root common.Hash) { +func (db *Database) Dereference(root common.Hash, prune bool) error { // Sanity check to ensure that the meta-root is not removed if root == (common.Hash{}) { log.Error("Attempted to dereference the trie cache meta root") - return + return nil } db.lock.Lock() defer db.lock.Unlock() nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() - db.dereference(root, common.Hash{}) - + prunetime, prunenodes, prunesize := db.prunetime, db.prunenodes, db.prunesize + if err := db.dereference(common.Hash{}, root, common.Hash{}, common.Hash{}, prune, nil, make(map[string]node)); err != nil { + return err + } db.gcnodes += uint64(nodes - len(db.dirties)) db.gcsize += storage - db.dirtiesSize db.gctime += time.Since(start) @@ -505,53 +583,220 @@ func (db *Database) Dereference(root common.Hash) { memcacheGCSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheGCNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) + memcachePruneTimeTimer.Update(db.prunetime - prunetime) + memcachePruneNodesMeter.Mark(int64(db.prunenodes - prunenodes)) + memcachePruneSizeMeter.Mark(int64(db.prunesize - prunesize)) + + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", common.PrettyDuration(time.Since(start)), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", common.PrettyDuration(db.gctime), "prnodes", db.prunenodes, "prsize", db.prunesize, "prtime", common.PrettyDuration(db.prunetime), + "livenodes", len(db.dirties), "livesize", db.dirtiesSize) + + return nil } // dereference is the private locked version of Dereference. -func (db *Database) dereference(child common.Hash, parent common.Hash) { +func (db *Database) dereference(childOwner common.Hash, childHash common.Hash, parentOwner common.Hash, parentHash common.Hash, prune bool, path []byte, cache map[string]node) error { // Dereference the parent-child - node := db.dirties[parent] + parentKey := makeNodeKey(parentOwner, parentHash) + parent := db.dirties[parentKey] - if node.children != nil && node.children[child] > 0 { - node.children[child]-- - if node.children[child] == 0 { - delete(node.children, child) + childKey := makeNodeKey(childOwner, childHash) + if parent.children != nil && parent.children[childKey] > 0 { + parent.children[childKey]-- + if parent.children[childKey] == 0 { + delete(parent.children, childKey) } } // If the child does not exist, it's a previously committed node. - node, ok := db.dirties[child] + child, ok := db.dirties[childKey] if !ok { - return + if prune { + batch := db.diskdb.NewBatch() + + start := time.Now() + db.prune(childOwner, childHash, path, batch, cache) + db.prunetime += time.Since(start) + + if err := batch.Write(); err != nil { + return err + } + } + return nil } // If there are no more references to the child, delete it and cascade - if node.parents > 0 { + if child.parents > 0 { // This is a special cornercase where a node loaded from disk (i.e. not in the // memcache any more) gets reinjected as a new node (short node split into full, // then reverted into short), causing a cached node to have no parents. That is // no problem in itself, but don't make maxint parents out of it. - node.parents-- + child.parents-- } - if node.parents == 0 { + if child.parents == 0 { // Remove the node from the flush-list - switch child { + switch childKey { case db.oldest: - db.oldest = node.flushNext - db.dirties[node.flushNext].flushPrev = common.Hash{} + db.oldest = child.flushNext + db.dirties[child.flushNext].flushPrev = metaRoot case db.newest: - db.newest = node.flushPrev - db.dirties[node.flushPrev].flushNext = common.Hash{} + db.newest = child.flushPrev + db.dirties[child.flushPrev].flushNext = metaRoot default: - db.dirties[node.flushPrev].flushNext = node.flushNext - db.dirties[node.flushNext].flushPrev = node.flushPrev + db.dirties[child.flushPrev].flushNext = child.flushNext + db.dirties[child.flushNext].flushPrev = child.flushPrev } // Dereference all children and delete the node - for _, hash := range node.childs() { - db.dereference(hash, child) + child.iterateRefs(path, func(path []byte, hash common.Hash) error { + db.dereference(childOwner, hash, childOwner, childHash, prune, path, cache) + return nil + }) + for key := range child.children { + owner, hash := splitNodeKey(key) + db.dereference(owner, hash, childOwner, childHash, prune, nil, cache) } - delete(db.dirties, child) - db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) + delete(db.dirties, childKey) + db.dirtiesSize -= common.StorageSize(common.HashLength + int(child.size)) + } + return nil +} + +// prune deletes a trie node from disk if there are no more live references to +// it, cascading until all dangling nodes are removed. +func (db *Database) prune(owner common.Hash, hash common.Hash, path []byte, batch ethdb.Batch, cache map[string]node) { + // If the node is still live in the memory cache, it's still referenced so we + // can abort. This case is important when and old trie being pruned references + // a new node (maybe that node was recreted since), since currently live nodes + // are stored expanded, not as hashes. + key := makeNodeKey(owner, hash) + if db.dirties[key] != nil { + return + } + // Iterate over all the live tries in the cache and check node liveliness + for key := range db.dirties[metaRoot].children { + _, root := splitNodeKey(key) + + var paths [][]byte + if owner != (common.Hash{}) { + paths = [][]byte{keybytesToHex(owner[:])} + } + if db.live(hashNode(root[:]), owner, hash, append(paths, path), cache) { + return + } + } + // Dead node found, delete it from the database + dead := []byte(makeNodeKey(owner, hash)) + blob, err := db.diskdb.Get(dead) + if blob == nil || err != nil { + log.Error("Missing prune target", "owner", owner, "hash", hash, "path", fmt.Sprintf("%x", path)) + return + } + node := mustDecodeNode(hash[:], blob, 0) + + // Prune the node and its children if it's not a bytecode blob + db.cleans.Delete(key) + batch.Delete(dead) + db.prunenodes++ + db.prunesize += common.StorageSize(len(blob)) + + iterateRefs(node, path, func(path []byte, hash common.Hash) error { + db.prune(owner, hash, path, batch, cache) + return nil + }) +} + +// live descends in the trie and returns whether the given hash is part of the +// trie or not. +func (db *Database) live(root node, owner common.Hash, hash common.Hash, paths [][]byte, cache map[string]node) bool { + // If we reached the end of our path, it should be a hash node + if len(paths) == 1 && len(paths[0]) == 0 { + if have, ok := root.(hashNode); ok { + return common.BytesToHash(have) == hash + } + // Not a hash node? It rarely happens that a 32+ byte leaf short node gets + // turned into a 31- byte one, converting if from a hash node to an embedded + // one. Allow this case, but reject as a wrong path. + + // TODO(karalabe): get rid of this warning, only curiosity + log.Warn("Liveness check terminated on non-hash", "type", fmt.Sprintf("%T", root), "node", root.fstring("")) + + return false + } + // If we're at a hash node, expand before continuing + if n, ok := root.(hashNode); ok { + var ( + key string + hash = common.BytesToHash(n) + ) + if len(paths) > 1 { + key = makeNodeKey(common.Hash{}, hash) + } else { + key = makeNodeKey(owner, hash) + } + if enc, err := db.cleans.Get(key); err == nil && enc != nil { + root = mustDecodeNode(hash[:], enc, 0) + cache[key] = root + } else if node := db.dirties[key]; node != nil { + root = node.node + } else if node := cache[key]; node != nil { + root = node + } else { + blob, err := db.diskdb.Get([]byte(key)) + if blob == nil || err != nil { + panic(fmt.Sprintf("missing referenced node %x (searching for %x:%x at %x)", key, owner, hash, paths)) + } + root = mustDecodeNode(hash[:], blob, 0) + cache[key] = root + } + } + // If we reached an account node, extract the storage trie root to continue on + if len(paths) == 2 && len(paths[0]) == 0 { + if have, ok := root.(valueNode); ok { + var account struct { + Nonce uint64 + Balance *big.Int + Root common.Hash + CodeHash []byte + } + if err := rlp.DecodeBytes(have, &account); err != nil { + panic(err) + } + if account.Root == emptyRoot { + return false + } + return db.live(hashNode(account.Root[:]), owner, hash, paths[1:], cache) + } + panic(fmt.Sprintf("liveness check path swap terminated on non value node: %T", root)) + } + + // Descend into the trie following the specified path. This code segment must + // be able to handle both simplified raw nodes kept in this cache as well as + // cold nodes loaded directly from disk. + switch n := root.(type) { + case *rawShortNode: + if prefixLen(n.Key, paths[0]) == len(n.Key) { + return db.live(n.Val, owner, hash, append([][]byte{paths[0][len(n.Key):]}, paths[1:]...), cache) + } + return false + + case *shortNode: + if prefixLen(n.Key, paths[0]) == len(n.Key) { + return db.live(n.Val, owner, hash, append([][]byte{paths[0][len(n.Key):]}, paths[1:]...), cache) + } + return false + + case rawFullNode: + if child := n[paths[0][0]]; child != nil { + return db.live(child, owner, hash, append([][]byte{paths[0][1:]}, paths[1:]...), cache) + } + return false + + case *fullNode: + if child := n.Children[paths[0][0]]; child != nil { + return db.live(child, owner, hash, append([][]byte{paths[0][1:]}, paths[1:]...), cache) + } + return false + + default: + panic(fmt.Sprintf("unknown node type: %T", n)) } } @@ -577,7 +822,7 @@ func (db *Database) Cap(limit common.StorageSize) error { flushPreimages := db.preimagesSize > 4*1024*1024 if flushPreimages { for hash, preimage := range db.preimages { - if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil { + if err := batch.Put(db.preimageKey(hash[:]), preimage); err != nil { log.Error("Failed to commit preimage from trie database", "err", err) db.lock.RUnlock() return err @@ -593,10 +838,10 @@ func (db *Database) Cap(limit common.StorageSize) error { } // Keep committing nodes from the flush-list until we're below allowance oldest := db.oldest - for size > limit && oldest != (common.Hash{}) { + for size > limit && oldest != metaRoot { // Fetch the oldest referenced node and push into the batch node := db.dirties[oldest] - if err := batch.Put(oldest[:], node.rlp()); err != nil { + if err := batch.Put([]byte(oldest), node.rlp()); err != nil { db.lock.RUnlock() return err } @@ -639,8 +884,8 @@ func (db *Database) Cap(limit common.StorageSize) error { db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } - if db.oldest != (common.Hash{}) { - db.dirties[db.oldest].flushPrev = common.Hash{} + if db.oldest != metaRoot { + db.dirties[db.oldest].flushPrev = metaRoot } db.flushnodes += uint64(nodes - len(db.dirties)) db.flushsize += storage - db.dirtiesSize @@ -650,8 +895,8 @@ func (db *Database) Cap(limit common.StorageSize) error { memcacheFlushSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheFlushNodesMeter.Mark(int64(nodes - len(db.dirties))) - log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", time.Since(start), - "flushnodes", db.flushnodes, "flushsize", db.flushsize, "flushtime", db.flushtime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) + log.Debug("Persisted nodes from memory database", "nodes", nodes-len(db.dirties), "size", storage-db.dirtiesSize, "time", common.PrettyDuration(time.Since(start)), + "flnodes", db.flushnodes, "flsize", db.flushsize, "fltime", common.PrettyDuration(db.flushtime), "livenodes", len(db.dirties), "livesize", db.dirtiesSize) return nil } @@ -672,7 +917,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { // Move all of the accumulated preimages into a write batch for hash, preimage := range db.preimages { - if err := batch.Put(db.secureKey(hash[:]), preimage); err != nil { + if err := batch.Put(db.preimageKey(hash[:]), preimage); err != nil { log.Error("Failed to commit preimage from trie database", "err", err) db.lock.RUnlock() return err @@ -687,7 +932,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { } // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize - if err := db.commit(node, batch); err != nil { + if err := db.commit(common.Hash{}, node, batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() return err @@ -707,7 +952,7 @@ func (db *Database) Commit(node common.Hash, report bool) error { db.preimages = make(map[common.Hash][]byte) db.preimagesSize = 0 - db.uncache(node) + db.uncache(common.Hash{}, node) memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) @@ -717,29 +962,39 @@ func (db *Database) Commit(node common.Hash, report bool) error { if !report { logger = log.Debug } - logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", time.Since(start)+db.flushtime, - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.dirties), "livesize", db.dirtiesSize) + logger("Persisted trie from memory database", "nodes", nodes-len(db.dirties)+int(db.flushnodes), "size", storage-db.dirtiesSize+db.flushsize, "time", common.PrettyDuration(time.Since(start)+db.flushtime), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", common.PrettyDuration(db.gctime), "prnodes", db.prunenodes, "prsize", db.prunesize, "prtime", common.PrettyDuration(db.prunetime), + "linodes", len(db.dirties), "lisize", db.dirtiesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 + db.prunenodes, db.prunesize, db.prunetime = 0, 0, 0 db.flushnodes, db.flushsize, db.flushtime = 0, 0, 0 return nil } // commit is the private locked version of Commit. -func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { +func (db *Database) commit(owner common.Hash, hash common.Hash, batch ethdb.Batch) error { // If the node does not exist, it's a previously committed node - node, ok := db.dirties[hash] + key := makeNodeKey(owner, hash) + + node, ok := db.dirties[key] if !ok { return nil } - for _, child := range node.childs() { - if err := db.commit(child, batch); err != nil { + if err := node.iterateRefs(nil, func(path []byte, child common.Hash) error { + return db.commit(owner, child, batch) + }); err != nil { + return err + } + for child := range node.children { + owner, hash := splitNodeKey(child) + if err := db.commit(owner, hash, batch); err != nil { return err } } - if err := batch.Put(hash[:], node.rlp()); err != nil { + if err := batch.Put([]byte(key), node.rlp()); err != nil { return err } // If we've reached an optimal batch size, commit and start over @@ -756,29 +1011,35 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { // persisted trie is removed from the cache. The reason behind the two-phase // commit is to ensure consistent data availability while moving from memory // to disk. -func (db *Database) uncache(hash common.Hash) { +func (db *Database) uncache(owner common.Hash, hash common.Hash) { // If the node does not exist, we're done on this path - node, ok := db.dirties[hash] + key := makeNodeKey(owner, hash) + + node, ok := db.dirties[key] if !ok { return } // Node still exists, remove it from the flush-list - switch hash { + switch key { case db.oldest: db.oldest = node.flushNext - db.dirties[node.flushNext].flushPrev = common.Hash{} + db.dirties[node.flushNext].flushPrev = metaRoot case db.newest: db.newest = node.flushPrev - db.dirties[node.flushPrev].flushNext = common.Hash{} + db.dirties[node.flushPrev].flushNext = metaRoot default: db.dirties[node.flushPrev].flushNext = node.flushNext db.dirties[node.flushNext].flushPrev = node.flushPrev } // Uncache the node's subtries and remove the node itself too - for _, child := range node.childs() { - db.uncache(child) + node.iterateRefs(nil, func(path []byte, child common.Hash) error { + db.uncache(owner, child) + return nil + }) + for child := range node.children { + db.uncache(splitNodeKey(child)) } - delete(db.dirties, hash) + delete(db.dirties, key) db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) } @@ -803,17 +1064,18 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // This method is extremely CPU and memory intensive, only use when must. func (db *Database) verifyIntegrity() { // Iterate over all the cached nodes and accumulate them into a set - reachable := map[common.Hash]struct{}{{}: {}} + reachable := map[string]struct{}{metaRoot: struct{}{}} - for child := range db.dirties[common.Hash{}].children { - db.accumulate(child, reachable) + for key := range db.dirties[metaRoot].children { + _, root := splitNodeKey(key) + db.accumulate(common.Hash{}, root, reachable) } // Find any unreachable but cached nodes var unreachable []string - for hash, node := range db.dirties { - if _, ok := reachable[hash]; !ok { + for key, node := range db.dirties { + if _, ok := reachable[key]; !ok { unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}", - hash, node.node, node.parents, node.flushPrev, node.flushNext)) + key, node.node, node.parents, node.flushPrev, node.flushNext)) } } if len(unreachable) != 0 { @@ -821,18 +1083,25 @@ func (db *Database) verifyIntegrity() { } } -// accumulate iterates over the trie defined by hash and accumulates all the -// cached children found in memory. -func (db *Database) accumulate(hash common.Hash, reachable map[common.Hash]struct{}) { +// accumulate iterates over the trie defined by owner:hash and accumulates all +// the cached children found in memory. +func (db *Database) accumulate(owner common.Hash, hash common.Hash, reachable map[string]struct{}) { // Mark the node reachable if present in the memory cache - node, ok := db.dirties[hash] + key := makeNodeKey(owner, hash) + + node, ok := db.dirties[key] if !ok { return } - reachable[hash] = struct{}{} + reachable[key] = struct{}{} // Iterate over all the children and accumulate them too - for _, child := range node.childs() { - db.accumulate(child, reachable) + node.iterateRefs(nil, func(path []byte, hash common.Hash) error { + db.accumulate(owner, hash, reachable) + return nil + }) + for key := range node.children { + owner, hash := splitNodeKey(key) + db.accumulate(owner, hash, reachable) } } diff --git a/trie/hasher.go b/trie/hasher.go index 9d6756b6f4..77c44094bb 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -31,6 +31,7 @@ type hasher struct { cachegen uint16 cachelimit uint16 onleaf LeafCallback + owner common.Hash } // keccakState wraps sha3.state. In addition to the usual hash methods, it also supports @@ -62,9 +63,9 @@ var hasherPool = sync.Pool{ }, } -func newHasher(cachegen, cachelimit uint16, onleaf LeafCallback) *hasher { +func newHasher(owner common.Hash, cachegen, cachelimit uint16, onleaf LeafCallback) *hasher { h := hasherPool.Get().(*hasher) - h.cachegen, h.cachelimit, h.onleaf = cachegen, cachelimit, onleaf + h.owner, h.cachegen, h.cachelimit, h.onleaf = owner, cachegen, cachelimit, onleaf return h } @@ -74,7 +75,7 @@ func returnHasherToPool(h *hasher) { // hash collapses a node down into a hash node, also returning a copy of the // original node initialized with the computed hash to replace the original one. -func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { +func (h *hasher) hash(path []byte, n node, db *Database, force bool) (node, node, error) { // If we're not storing the node, just hashing, use available cached data if hash, dirty := n.cache(); hash != nil { if db == nil { @@ -91,11 +92,11 @@ func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { } } // Trie not processed yet or needs storage, walk the children - collapsed, cached, err := h.hashChildren(n, db) + collapsed, cached, err := h.hashChildren(path, n, db) if err != nil { return hashNode{}, n, err } - hashed, err := h.store(collapsed, db, force) + hashed, err := h.store(path, collapsed, db, force) if err != nil { return hashNode{}, n, err } @@ -121,7 +122,7 @@ func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { // hashChildren replaces the children of a node with their hashes if the encoded // size of the child is larger than a hash, returning the collapsed node as well // as a replacement for the original node with the child hashes cached in. -func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { +func (h *hasher) hashChildren(path []byte, original node, db *Database) (node, node, error) { var err error switch n := original.(type) { @@ -132,7 +133,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { cached.Key = common.CopyBytes(n.Key) if _, ok := n.Val.(valueNode); !ok { - collapsed.Val, cached.Val, err = h.hash(n.Val, db, false) + collapsed.Val, cached.Val, err = h.hash(append(path, n.Key...), n.Val, db, false) if err != nil { return original, original, err } @@ -145,7 +146,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { for i := 0; i < 16; i++ { if n.Children[i] != nil { - collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) + collapsed.Children[i], cached.Children[i], err = h.hash(append(path, byte(i)), n.Children[i], db, false) if err != nil { return original, original, err } @@ -163,7 +164,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { // store hashes the node n and if we have a storage layer specified, it writes // the key/value pair to it and tracks any node->child references as well as any // node->external trie references. -func (h *hasher) store(n node, db *Database, force bool) (node, error) { +func (h *hasher) store(path []byte, n node, db *Database, force bool) (node, error) { // Don't store hashes or empty nodes. if _, isHash := n.(hashNode); n == nil || isHash { return n, nil @@ -187,7 +188,7 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { hash := common.BytesToHash(hash) db.lock.Lock() - db.insert(hash, h.tmp, n) + db.insert(h.owner, hash, h.tmp, n) db.lock.Unlock() // Track external references from account->storage trie @@ -195,12 +196,12 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) { switch n := n.(type) { case *shortNode: if child, ok := n.Val.(valueNode); ok { - h.onleaf(child, hash) + h.onleaf(common.BytesToHash(hexToKeybytes(append(path, compactToHex(n.Key)...))), child, hash) } case *fullNode: for i := 0; i < 16; i++ { if child, ok := n.Children[i].(valueNode); ok { - h.onleaf(child, hash) + h.onleaf(common.BytesToHash(hexToKeybytes(append(path, byte(i)))), child, hash) } } } diff --git a/trie/iterator.go b/trie/iterator.go index 77f1681665..51e2f1e3b2 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -180,15 +180,14 @@ func (it *nodeIterator) LeafBlob() []byte { func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - hasher := newHasher(0, 0, nil) + hasher := newHasher(common.Hash{}, 0, 0, nil) defer returnHasherToPool(hasher) proofs := make([][]byte, 0, len(it.stack)) - for i, item := range it.stack[:len(it.stack)-1] { // Gather nodes that end up as hash nodes (or the root) - node, _, _ := hasher.hashChildren(item.node, nil) - hashed, _ := hasher.store(node, nil, false) + node, _, _ := hasher.hashChildren(nil, item.node, nil) + hashed, _ := hasher.store(nil, node, nil, false) if _, ok := hashed.(hashNode); ok || i == 0 { enc, _ := rlp.EncodeToBytes(node) proofs = append(proofs, enc) diff --git a/trie/proof.go b/trie/proof.go index 1334bde970..1eafc5873a 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -65,14 +65,14 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error { panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - hasher := newHasher(0, 0, nil) + hasher := newHasher(common.Hash{}, 0, 0, nil) defer returnHasherToPool(hasher) for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. - n, _, _ = hasher.hashChildren(n, nil) - hn, _ := hasher.store(n, nil, false) + n, _, _ = hasher.hashChildren(nil, n, nil) + hn, _ := hasher.store(nil, n, nil, false) if hash, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 6a50cfd5a6..507dcca846 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -52,10 +52,26 @@ type SecureTrie struct { // A new cache generation is created by each call to Commit. // cachelimit sets the number of past cache generations to keep. func NewSecure(root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) { + return NewSecureWithOwner(common.Hash{}, root, db, cachelimit) +} + +// NewSecureWithOwner creates a trie with an existing root node from a backing +// database with an assigned owner for storage proximity and optional intermediate +// in-memory node pool. +// +// If root is the zero hash or the sha3 hash of an empty string, the +// trie is initially empty. Otherwise, New will panic if db is nil +// and returns MissingNodeError if the root node cannot be found. +// +// Accessing the trie loads nodes from the database or node pool on demand. +// Loaded nodes are kept around until their 'cache generation' expires. +// A new cache generation is created by each call to Commit. +// cachelimit sets the number of past cache generations to keep. +func NewSecureWithOwner(owner common.Hash, root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) { if db == nil { panic("trie.NewSecure called without a database") } - trie, err := New(root, db) + trie, err := NewWithOwner(owner, root, db) if err != nil { return nil, err } @@ -183,7 +199,7 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { - h := newHasher(0, 0, nil) + h := newHasher(t.trie.owner, 0, 0, nil) h.sha.Reset() h.sha.Write(key) buf := h.sha.Sum(t.hashKeyBuf[:0]) diff --git a/trie/sync.go b/trie/sync.go index 44f5087b9f..738fd79d59 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -280,7 +280,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // Notify any external watcher of a new key/value node if req.callback != nil { if node, ok := (child.node).(valueNode); ok { - if err := req.callback(node, req.hash); err != nil { + if err := req.callback(common.Hash{}, node, req.hash); err != nil { return nil, err } } diff --git a/trie/trie.go b/trie/trie.go index af424d4ac6..f32d5dd591 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -57,7 +57,7 @@ func CacheUnloads() int64 { // LeafCallback is a callback type invoked when a trie operation reaches a leaf // node. It's used by state sync and commit to allow handling external references // between account and storage tries. -type LeafCallback func(leaf []byte, parent common.Hash) error +type LeafCallback func(owner common.Hash, leaf []byte, parent common.Hash) error // Trie is a Merkle Patricia Trie. // The zero value is an empty trie with no database. @@ -65,8 +65,9 @@ type LeafCallback func(leaf []byte, parent common.Hash) error // // Trie is not safe for concurrent use. type Trie struct { - db *Database - root node + db *Database + root node + owner common.Hash // Cache generation values. // cachegen increases by one with each commit operation. @@ -93,11 +94,23 @@ func (t *Trie) newFlag() nodeFlag { // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. func New(root common.Hash, db *Database) (*Trie, error) { + return NewWithOwner(common.Hash{}, root, db) +} + +// NewWithOwner creates a trie with an existing root node from db and an assigned +// owner for storage proximity. +// +// If root is the zero hash or the sha3 hash of an empty string, the +// trie is initially empty and does not require a database. Otherwise, +// New will panic if db is nil and returns a MissingNodeError if root does +// not exist in the database. Accessing the trie loads nodes from db on demand. +func NewWithOwner(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { if db == nil { panic("trie.New called without a database") } trie := &Trie{ - db: db, + db: db, + owner: owner, } if root != (common.Hash{}) && root != emptyRoot { rootnode, err := trie.resolveHash(root[:], nil) @@ -431,9 +444,10 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { cacheMissCounter.Inc(1) hash := common.BytesToHash(n) - if node := t.db.node(hash, t.cachegen); node != nil { + if node := t.db.node(t.owner, hash, t.cachegen); node != nil { return node, nil } + log.Warn("Missing trie node", "owner", t.owner.Hex(), "hash", hash.Hex(), "path", fmt.Sprintf("%x", prefix)) return nil, &MissingNodeError{NodeHash: hash, Path: prefix} } @@ -468,7 +482,7 @@ func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil, nil } - h := newHasher(t.cachegen, t.cachelimit, onleaf) + h := newHasher(t.owner, t.cachegen, t.cachelimit, onleaf) defer returnHasherToPool(h) - return h.hash(t.root, db, true) + return h.hash(nil, t.root, db, true) }