forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 02:12:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core, eth, light, trie: db storage reorg benchmark hack

Browse source
This commit is contained in:
Péter Szilágyi 2018-05-24 12:20:19 +03:00
parent fbf57d53e2
commit 0b297787ac
No known key found for this signature in database
GPG key ID: E9AE538CEDF8293D
22 changed files with 199 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
core/blockchain.go
View file

@ -319,7 +319,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
if block == nil { if block == nil {
return fmt.Errorf("non existent block [%x…]", hash[:4]) return fmt.Errorf("non existent block [%x…]", hash[:4])
} }
if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil { if _, err := trie.NewSecure(nil, block.Root(), bc.stateCache.TrieDB(), 0); err != nil {
return err return err
} }
// If all checks out, manually set the head block // If all checks out, manually set the head block
@ -636,8 +636,8 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.
// TrieNode retrieves a blob of data associated with a trie node (or code hash) // TrieNode retrieves a blob of data associated with a trie node (or code hash)
// either from ephemeral in-memory cache, or from persistent storage. // either from ephemeral in-memory cache, or from persistent storage.
func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { func (bc *BlockChain) TrieNode(prefix []byte, hash common.Hash) ([]byte, error) {
return bc.stateCache.TrieDB().Node(hash) return bc.stateCache.TrieDB().Node(prefix, hash)
} }
// Stop stops the blockchain service. If any imports are currently in progress // Stop stops the blockchain service. If any imports are currently in progress
@ -666,13 +666,13 @@ func (bc *BlockChain) Stop() {
recent := bc.GetBlockByNumber(number - offset) recent := bc.GetBlockByNumber(number - offset)
log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
if err := triedb.Commit(recent.Root(), true); err != nil { if err := triedb.Commit(nil, recent.Root(), true); err != nil {
log.Error("Failed to commit recent state trie", "err", err) log.Error("Failed to commit recent state trie", "err", err)
} }
} }
} }
for !bc.triegc.Empty() { for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem().(common.Hash), common.Hash{}) triedb.Dereference(nil, bc.triegc.PopItem().(common.Hash), nil, common.Hash{})
} }
if size := triedb.Size(); size != 0 { if size := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup") log.Error("Dangling trie nodes after full cleanup")
@ -907,12 +907,12 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
// If we're running an archive node, always flush // If we're running an archive node, always flush
if bc.cacheConfig.Disabled { if bc.cacheConfig.Disabled {
if err := triedb.Commit(root, false); err != nil { if err := triedb.Commit(nil, root, false); err != nil {
return NonStatTy, err return NonStatTy, err
} }
} else { } else {
// Full but not archive node, do proper garbage collection // Full but not archive node, do proper garbage collection
triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive triedb.Reference(nil, root, nil, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -float32(block.NumberU64())) bc.triegc.Push(root, -float32(block.NumberU64()))
if current := block.NumberU64(); current > triesInMemory { if current := block.NumberU64(); current > triesInMemory {
@ -939,7 +939,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
} }
// If optimum or critical limits reached, write to disk // If optimum or critical limits reached, write to disk
if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
triedb.Commit(header.Root, true) triedb.Commit(nil, header.Root, true)
lastWrite = chosen lastWrite = chosen
bc.gcproc = 0 bc.gcproc = 0
} }
@ -951,7 +951,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
bc.triegc.Push(root, number) bc.triegc.Push(root, number)
break break
} }
triedb.Dereference(root.(common.Hash), common.Hash{}) triedb.Dereference(nil, root.(common.Hash), nil, common.Hash{})
} }
} }
} }

2
core/chain_makers.go
View file

@ -208,7 +208,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
if err != nil { if err != nil {
panic(fmt.Sprintf("state write error: %v", err)) panic(fmt.Sprintf("state write error: %v", err))
} }
if err := statedb.Database().TrieDB().Commit(root, false); err != nil { if err := statedb.Database().TrieDB().Commit(nil, root, false); err != nil {
panic(fmt.Sprintf("trie write error: %v", err)) panic(fmt.Sprintf("trie write error: %v", err))
} }
return block, b.receipts return block, b.receipts

2
core/genesis.go
View file

@ -254,7 +254,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
head.Difficulty = params.GenesisDifficulty head.Difficulty = params.GenesisDifficulty
} }
statedb.Commit(false) statedb.Commit(false)
statedb.Database().TrieDB().Commit(root, true) statedb.Database().TrieDB().Commit(nil, root, true)
return types.NewBlock(head, nil, nil, nil) return types.NewBlock(head, nil, nil, nil)
} }

6
core/state/database.go
View file

@ -100,7 +100,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
return cachedTrie{db.pastTries[i].Copy(), db}, nil return cachedTrie{db.pastTries[i].Copy(), db}, nil
} }
} }
tr, err := trie.NewSecure(root, db.db, MaxTrieCacheGen) tr, err := trie.NewSecure(nil, root, db.db, MaxTrieCacheGen)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -121,7 +121,7 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
// OpenStorageTrie opens the storage trie of an account. // OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
return trie.NewSecure(root, db.db, 0) return trie.NewSecure(addrHash[:], root, db.db, 0)
} }
// CopyTrie returns an independent copy of the given trie. // CopyTrie returns an independent copy of the given trie.
@ -138,7 +138,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
// ContractCode retrieves a particular contract's code. // ContractCode retrieves a particular contract's code.
func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
code, err := db.db.Node(codeHash) code, err := db.db.Node(nil, codeHash)
if err == nil { if err == nil {
db.codeSizeCache.Add(codeHash, len(code)) db.codeSizeCache.Add(codeHash, len(code))
} }

8
core/state/statedb.go
View file

@ -596,7 +596,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
case isDirty: case isDirty:
// Write any contract code associated with the state object // Write any contract code associated with the state object
if stateObject.code != nil && stateObject.dirtyCode { if stateObject.code != nil && stateObject.dirtyCode {
s.db.TrieDB().Insert(common.BytesToHash(stateObject.CodeHash()), stateObject.code) s.db.TrieDB().Insert(nil, common.BytesToHash(stateObject.CodeHash()), stateObject.code)
stateObject.dirtyCode = false stateObject.dirtyCode = false
} }
// Write any storage changes in the state object to its storage trie. // Write any storage changes in the state object to its storage trie.
@ -609,17 +609,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
delete(s.stateObjectsDirty, addr) delete(s.stateObjectsDirty, addr)
} }
// Write trie changes. // Write trie changes.
root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error { root, err = s.trie.Commit(func(path []byte, leaf []byte, parent common.Hash) error {
var account Account var account Account
if err := rlp.DecodeBytes(leaf, &account); err != nil { if err := rlp.DecodeBytes(leaf, &account); err != nil {
return nil return nil
} }
if account.Root != emptyState { if account.Root != emptyState {
s.db.TrieDB().Reference(account.Root, parent) s.db.TrieDB().Reference(path, account.Root, nil, parent)
} }
code := common.BytesToHash(account.CodeHash) code := common.BytesToHash(account.CodeHash)
if code != emptyCode { if code != emptyCode {
s.db.TrieDB().Reference(code, parent) s.db.TrieDB().Reference(nil, code, nil, parent)
} }
return nil return nil
}) })

2
core/state/sync.go
View file

@ -27,7 +27,7 @@ import (
// NewStateSync create a new state trie download scheduler. // NewStateSync create a new state trie download scheduler.
func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync { func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync {
var syncer *trie.TrieSync var syncer *trie.TrieSync
callback := func(leaf []byte, parent common.Hash) error { callback := func(path []byte, leaf []byte, parent common.Hash) error {
var obj Account var obj Account
if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil {
return err return err

4
eth/api.go
View file

@ -466,11 +466,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
} }
oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) oldTrie, err := trie.NewSecure(nil, startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }
newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0) newTrie, err := trie.NewSecure(nil, endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
if err != nil { if err != nil {
return nil, err return nil, err
} }

12
eth/api_tracer.go
View file

@ -291,12 +291,12 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
break break
} }
// Reference the trie twice, once for us, once for the trancer // Reference the trie twice, once for us, once for the trancer
database.TrieDB().Reference(root, common.Hash{}) database.TrieDB().Reference(nil, root, nil, common.Hash{})
if number >= origin { if number >= origin {
database.TrieDB().Reference(root, common.Hash{}) database.TrieDB().Reference(nil, root, nil, common.Hash{})
} }
// Dereference all past tries we ourselves are done working with // Dereference all past tries we ourselves are done working with
database.TrieDB().Dereference(proot, common.Hash{}) database.TrieDB().Dereference(nil, proot, nil, common.Hash{})
proot = root proot = root
} }
}() }()
@ -317,7 +317,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
done[uint64(result.Block)] = result done[uint64(result.Block)] = result
// Dereference any paret tries held in memory by this task // Dereference any paret tries held in memory by this task
database.TrieDB().Dereference(res.rootref, common.Hash{}) database.TrieDB().Dereference(nil, res.rootref, nil, common.Hash{})
// Stream completed traces to the user, aborting on the first error // Stream completed traces to the user, aborting on the first error
for result, ok := done[next]; ok; result, ok = done[next] { for result, ok := done[next]; ok; result, ok = done[next] {
@ -522,8 +522,8 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
if err := statedb.Reset(root); err != nil { if err := statedb.Reset(root); err != nil {
return nil, err return nil, err
} }
database.TrieDB().Reference(root, common.Hash{}) database.TrieDB().Reference(nil, root, nil, common.Hash{})
database.TrieDB().Dereference(proot, common.Hash{}) database.TrieDB().Dereference(nil, proot, nil, common.Hash{})
proot = root proot = root
} }
log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", database.TrieDB().Size()) log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", database.TrieDB().Size())

2
eth/handler.go
View file

@ -537,7 +537,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err) return errResp(ErrDecode, "msg %v: %v", msg, err)
} }
// Retrieve the requested state entry, stopping if enough was found // Retrieve the requested state entry, stopping if enough was found
if entry, err := pm.blockchain.TrieNode(hash); err == nil { if entry, err := pm.blockchain.TrieNode(nil, hash); err == nil {
data = append(data, entry) data = append(data, entry)
bytes += len(entry) bytes += len(entry)
} }

8
les/handler.go
View file

@ -592,7 +592,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if err != nil { if err != nil {
continue continue
} }
code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) code, _ := statedb.Database().TrieDB().Node(nil, common.BytesToHash(account.CodeHash))
data = append(data, code) data = append(data, code)
if bytes += len(code); bytes >= softResponseLimit { if bytes += len(code); bytes >= softResponseLimit {
@ -873,7 +873,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1)
if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) {
trie, err := trie.New(root, trieDb) trie, err := trie.New(nil, root, trieDb)
if err != nil { if err != nil {
continue continue
} }
@ -927,7 +927,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
var prefix string var prefix string
if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) {
auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) auxTrie, _ = trie.New(nil, root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix)))
} }
} }
if req.AuxReq == auxRoot { if req.AuxReq == auxRoot {
@ -1107,7 +1107,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
// getAccount retrieves an account from the state based at root. // getAccount retrieves an account from the state based at root.
func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common.Hash) (state.Account, error) { func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common.Hash) (state.Account, error) {
trie, err := trie.New(root, statedb.Database().TrieDB()) trie, err := trie.New(nil, root, statedb.Database().TrieDB())
if err != nil { if err != nil {
return state.Account{}, err return state.Account{}, err
} }

8
light/postprocess.go
View file

@ -152,7 +152,7 @@ func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) e
root = GetChtRoot(c.diskdb, section-1, lastSectionHead) root = GetChtRoot(c.diskdb, section-1, lastSectionHead)
} }
var err error var err error
c.trie, err = trie.New(root, c.triedb) c.trie, err = trie.New(nil, root, c.triedb)
c.section = section c.section = section
return err return err
} }
@ -178,7 +178,7 @@ func (c *ChtIndexerBackend) Commit() error {
if err != nil { if err != nil {
return err return err
} }
c.triedb.Commit(root, false) c.triedb.Commit(nil, root, false)
if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 {
log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root)
@ -250,7 +250,7 @@ func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.H
root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead)
} }
var err error var err error
b.trie, err = trie.New(root, b.triedb) b.trie, err = trie.New(nil, root, b.triedb)
b.section = section b.section = section
return err return err
} }
@ -297,7 +297,7 @@ func (b *BloomTrieIndexerBackend) Commit() error {
if err != nil { if err != nil {
return err return err
} }
b.triedb.Commit(root, false) b.triedb.Commit(nil, root, false)
sectionHead := b.sectionHeads[b.bloomTrieRatio-1] sectionHead := b.sectionHeads[b.bloomTrieRatio-1]
log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize))

4
light/trie.go
View file

@ -151,7 +151,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
for { for {
var err error var err error
if t.trie == nil { if t.trie == nil {
t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) t.trie, err = trie.New(nil, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
} }
if err == nil { if err == nil {
err = fn() err = fn()
@ -177,7 +177,7 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
// Open the actual non-ODR trie if that hasn't happened yet. // Open the actual non-ODR trie if that hasn't happened yet.
if t.trie == nil { if t.trie == nil {
it.do(func() error { it.do(func() error {
t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) t, err := trie.New(nil, t.id.Root, trie.NewDatabase(t.db.backend.Database()))
if err == nil { if err == nil {
it.t.trie = t it.t.trie = t
} }

96
trie/database.go
View file

@ -46,7 +46,7 @@ type DatabaseReader interface {
type Database struct { type Database struct {
diskdb ethdb.Database // Persistent storage for matured trie nodes diskdb ethdb.Database // Persistent storage for matured trie nodes
nodes map[common.Hash]*cachedNode // Data and references relationships of a node nodes map[string]*cachedNode // Data and references relationships of a node
preimages map[common.Hash][]byte // Preimages of nodes from the secure trie preimages map[common.Hash][]byte // Preimages of nodes from the secure trie
seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys seckeybuf [secureKeyLength]byte // Ephemeral buffer for calculating preimage keys
@ -65,7 +65,7 @@ type Database struct {
type cachedNode struct { type cachedNode struct {
blob []byte // Cached data block of the trie node blob []byte // Cached data block of the trie node
parents int // Number of live nodes referencing this one parents int // Number of live nodes referencing this one
children map[common.Hash]int // Children referenced by this nodes children map[string]int // Children referenced by this nodes
} }
// NewDatabase creates a new trie database to store ephemeral trie content before // NewDatabase creates a new trie database to store ephemeral trie content before
@ -73,8 +73,8 @@ type cachedNode struct {
func NewDatabase(diskdb ethdb.Database) *Database { func NewDatabase(diskdb ethdb.Database) *Database {
return &Database{ return &Database{
diskdb: diskdb, diskdb: diskdb,
nodes: map[common.Hash]*cachedNode{ nodes: map[string]*cachedNode{
{}: {children: make(map[common.Hash]int)}, dbkey(nil, common.Hash{}): {children: make(map[string]int)},
}, },
preimages: make(map[common.Hash][]byte), preimages: make(map[common.Hash][]byte),
} }
@ -87,23 +87,29 @@ func (db *Database) DiskDB() DatabaseReader {
// Insert writes a new trie node to the memory database if it's yet unknown. The // Insert writes a new trie node to the memory database if it's yet unknown. The
// method will make a copy of the slice. // method will make a copy of the slice.
func (db *Database) Insert(hash common.Hash, blob []byte) { func (db *Database) Insert(path []byte, hash common.Hash, blob []byte) {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
db.insert(hash, blob) db.insert(path, hash, blob)
}
// dbkey returns the database key corresponding to a particular trie path and hash.
func dbkey(path []byte, hash common.Hash) string {
return string(append(path, hash[:]...))
} }
// insert is the private locked version of Insert. // insert is the private locked version of Insert.
func (db *Database) insert(hash common.Hash, blob []byte) { func (db *Database) insert(path []byte, hash common.Hash, blob []byte) {
if _, ok := db.nodes[hash]; ok { key := dbkey(path, hash)
if _, ok := db.nodes[key]; ok {
return return
} }
db.nodes[hash] = &cachedNode{ db.nodes[key] = &cachedNode{
blob: common.CopyBytes(blob), blob: common.CopyBytes(blob),
children: make(map[common.Hash]int), children: make(map[string]int),
} }
db.nodesSize += common.StorageSize(common.HashLength + len(blob)) db.nodesSize += common.StorageSize(len(key) + len(blob))
} }
// insertPreimage writes a new trie node pre-image to the memory database if it's // insertPreimage writes a new trie node pre-image to the memory database if it's
@ -120,17 +126,19 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
// Node retrieves a cached trie node from memory. If it cannot be found cached, // Node retrieves a cached trie node from memory. If it cannot be found cached,
// the method queries the persistent database for the content. // the method queries the persistent database for the content.
func (db *Database) Node(hash common.Hash) ([]byte, error) { func (db *Database) Node(path []byte, hash common.Hash) ([]byte, error) {
// Retrieve the node from cache if available // Retrieve the node from cache if available
key := dbkey(path, hash)
db.lock.RLock() db.lock.RLock()
node := db.nodes[hash] node := db.nodes[key]
db.lock.RUnlock() db.lock.RUnlock()
if node != nil { if node != nil {
return node.blob, nil return node.blob, nil
} }
// Content unavailable in memory, attempt to retrieve from disk // Content unavailable in memory, attempt to retrieve from disk
return db.diskdb.Get(hash[:]) return db.diskdb.Get([]byte(key))
} }
// preimage retrieves a cached trie node pre-image from memory. If it cannot be // preimage retrieves a cached trie node pre-image from memory. If it cannot be
@ -157,39 +165,45 @@ func (db *Database) secureKey(key []byte) []byte {
return buf return buf
} }
// Nodes retrieves the hashes of all the nodes cached within the memory database. // Nodes retrieves the keys of all the nodes cached within the memory database.
// This method is extremely expensive and should only be used to validate internal // This method is extremely expensive and should only be used to validate internal
// states in test code. // states in test code.
func (db *Database) Nodes() []common.Hash { func (db *Database) Nodes() []string {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
var hashes = make([]common.Hash, 0, len(db.nodes)) var keys = make([]string, 0, len(db.nodes))
for hash := range db.nodes {
var hash common.Hash
for key := range db.nodes {
copy(hash[:], key[len(key)-common.HashLength:])
if hash != (common.Hash{}) { // Special case for "root" references/nodes if hash != (common.Hash{}) { // Special case for "root" references/nodes
hashes = append(hashes, hash) keys = append(keys, key)
} }
} }
return hashes return keys
} }
// Reference adds a new reference from a parent node to a child node. // Reference adds a new reference from a parent node to a child node.
func (db *Database) Reference(child common.Hash, parent common.Hash) { func (db *Database) Reference(childPath []byte, childHash common.Hash, parentPath []byte, parentHash common.Hash) {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
db.reference(child, parent) db.reference(childPath, childHash, parentPath, parentHash)
} }
// reference is the private locked version of Reference. // reference is the private locked version of Reference.
func (db *Database) reference(child common.Hash, parent common.Hash) { func (db *Database) reference(childPath []byte, childHash common.Hash, parentPath []byte, parentHash common.Hash) {
// If the node does not exist, it's a node pulled from disk, skip // If the node does not exist, it's a node pulled from disk, skip
child := dbkey(childPath, childHash)
node, ok := db.nodes[child] node, ok := db.nodes[child]
if !ok { if !ok {
return return
} }
// If the reference already exists, only duplicate for roots // If the reference already exists, only duplicate for roots
if _, ok = db.nodes[parent].children[child]; ok && parent != (common.Hash{}) { parent := dbkey(parentPath, parentHash)
if _, ok = db.nodes[parent].children[child]; ok && parentHash != (common.Hash{}) {
return return
} }
node.parents++ node.parents++
@ -197,12 +211,12 @@ func (db *Database) reference(child common.Hash, parent common.Hash) {
} }
// Dereference removes an existing reference from a parent node to a child node. // Dereference removes an existing reference from a parent node to a child node.
func (db *Database) Dereference(child common.Hash, parent common.Hash) { func (db *Database) Dereference(childPath []byte, childHash common.Hash, parentPath []byte, parentHash common.Hash) {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() nodes, storage, start := len(db.nodes), db.nodesSize, time.Now()
db.dereference(child, parent) db.dereference(dbkey(childPath, childHash), dbkey(parentPath, parentHash))
db.gcnodes += uint64(nodes - len(db.nodes)) db.gcnodes += uint64(nodes - len(db.nodes))
db.gcsize += storage - db.nodesSize db.gcsize += storage - db.nodesSize
@ -213,7 +227,7 @@ func (db *Database) Dereference(child common.Hash, parent common.Hash) {
} }
// dereference is the private locked version of Dereference. // dereference is the private locked version of Dereference.
func (db *Database) dereference(child common.Hash, parent common.Hash) { func (db *Database) dereference(child string, parent string) {
// Dereference the parent-child // Dereference the parent-child
node := db.nodes[parent] node := db.nodes[parent]
@ -229,11 +243,11 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) {
// If there are no more references to the child, delete it and cascade // If there are no more references to the child, delete it and cascade
node.parents-- node.parents--
if node.parents == 0 { if node.parents == 0 {
for hash := range node.children { for path := range node.children {
db.dereference(hash, child) db.dereference(path, child)
} }
delete(db.nodes, child) delete(db.nodes, child)
db.nodesSize -= common.StorageSize(common.HashLength + len(node.blob)) db.nodesSize -= common.StorageSize(len(child) + len(node.blob))
} }
} }
@ -241,7 +255,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) {
// to disk, forcefully tearing down all references in both directions. // to disk, forcefully tearing down all references in both directions.
// //
// As a side effect, all pre-images accumulated up to this point are also written. // As a side effect, all pre-images accumulated up to this point are also written.
func (db *Database) Commit(node common.Hash, report bool) error { func (db *Database) Commit(path []byte, node common.Hash, report bool) error {
// Create a database batch to flush persistent data out. It is important that // Create a database batch to flush persistent data out. It is important that
// outside code doesn't see an inconsistent state (referenced data removed from // outside code doesn't see an inconsistent state (referenced data removed from
// memory cache during commit but not yet in persistent storage). This is ensured // memory cache during commit but not yet in persistent storage). This is ensured
@ -267,7 +281,9 @@ func (db *Database) Commit(node common.Hash, report bool) error {
} }
// Move the trie itself into the batch, flushing if enough data is accumulated // Move the trie itself into the batch, flushing if enough data is accumulated
nodes, storage := len(db.nodes), db.nodesSize+db.preimagesSize nodes, storage := len(db.nodes), db.nodesSize+db.preimagesSize
if err := db.commit(node, batch); err != nil {
key := dbkey(path, node)
if err := db.commit(key, batch); err != nil {
log.Error("Failed to commit trie from trie database", "err", err) log.Error("Failed to commit trie from trie database", "err", err)
db.lock.RUnlock() db.lock.RUnlock()
return err return err
@ -287,7 +303,7 @@ func (db *Database) Commit(node common.Hash, report bool) error {
db.preimages = make(map[common.Hash][]byte) db.preimages = make(map[common.Hash][]byte)
db.preimagesSize = 0 db.preimagesSize = 0
db.uncache(node) db.uncache(key)
logger := log.Info logger := log.Info
if !report { if !report {
@ -303,9 +319,9 @@ func (db *Database) Commit(node common.Hash, report bool) error {
} }
// commit is the private locked version of Commit. // commit is the private locked version of Commit.
func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error { func (db *Database) commit(key string, batch ethdb.Batch) error {
// If the node does not exist, it's a previously committed node // If the node does not exist, it's a previously committed node
node, ok := db.nodes[hash] node, ok := db.nodes[key]
if !ok { if !ok {
return nil return nil
} }
@ -314,7 +330,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error {
return err return err
} }
} }
if err := batch.Put(hash[:], node.blob); err != nil { if err := batch.Put([]byte(key), node.blob); err != nil {
return err return err
} }
// If we've reached an optimal match size, commit and start over // If we've reached an optimal match size, commit and start over
@ -331,9 +347,9 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch) error {
// persisted trie is removed from the cache. The reason behind the two-phase // persisted trie is removed from the cache. The reason behind the two-phase
// commit is to ensure consistent data availability while moving from memory // commit is to ensure consistent data availability while moving from memory
// to disk. // to disk.
func (db *Database) uncache(hash common.Hash) { func (db *Database) uncache(key string) {
// If the node does not exist, we're done on this path // If the node does not exist, we're done on this path
node, ok := db.nodes[hash] node, ok := db.nodes[key]
if !ok { if !ok {
return return
} }
@ -341,8 +357,8 @@ func (db *Database) uncache(hash common.Hash) {
for child := range node.children { for child := range node.children {
db.uncache(child) db.uncache(child)
} }
delete(db.nodes, hash) delete(db.nodes, key)
db.nodesSize -= common.StorageSize(common.HashLength + len(node.blob)) db.nodesSize -= common.StorageSize(len(key) + len(node.blob))
} }
// Size returns the current storage size of the memory cache in front of the // Size returns the current storage size of the memory cache in front of the

28
trie/hasher.go
View file

@ -53,7 +53,7 @@ func returnHasherToPool(h *hasher) {
// hash collapses a node down into a hash node, also returning a copy of the // hash collapses a node down into a hash node, also returning a copy of the
// original node initialized with the computed hash to replace the original one. // original node initialized with the computed hash to replace the original one.
func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { func (h *hasher) hash(prefix []byte, path []byte, n node, db *Database, force bool) (node, node, error) {
// If we're not storing the node, just hashing, use available cached data // If we're not storing the node, just hashing, use available cached data
if hash, dirty := n.cache(); hash != nil { if hash, dirty := n.cache(); hash != nil {
if db == nil { if db == nil {
@ -70,11 +70,11 @@ func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) {
} }
} }
// Trie not processed yet or needs storage, walk the children // Trie not processed yet or needs storage, walk the children
collapsed, cached, err := h.hashChildren(n, db) collapsed, cached, err := h.hashChildren(prefix, path, n, db)
if err != nil { if err != nil {
return hashNode{}, n, err return hashNode{}, n, err
} }
hashed, err := h.store(collapsed, db, force) hashed, err := h.store(prefix, path, collapsed, db, force)
if err != nil { if err != nil {
return hashNode{}, n, err return hashNode{}, n, err
} }
@ -100,7 +100,7 @@ func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) {
// hashChildren replaces the children of a node with their hashes if the encoded // hashChildren replaces the children of a node with their hashes if the encoded
// size of the child is larger than a hash, returning the collapsed node as well // size of the child is larger than a hash, returning the collapsed node as well
// as a replacement for the original node with the child hashes cached in. // as a replacement for the original node with the child hashes cached in.
func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { func (h *hasher) hashChildren(prefix []byte, path []byte, original node, db *Database) (node, node, error) {
var err error var err error
switch n := original.(type) { switch n := original.(type) {
@ -111,7 +111,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) {
cached.Key = common.CopyBytes(n.Key) cached.Key = common.CopyBytes(n.Key)
if _, ok := n.Val.(valueNode); !ok { if _, ok := n.Val.(valueNode); !ok {
collapsed.Val, cached.Val, err = h.hash(n.Val, db, false) collapsed.Val, cached.Val, err = h.hash(prefix, append(path, n.Key...), n.Val, db, false)
if err != nil { if err != nil {
return original, original, err return original, original, err
} }
@ -127,7 +127,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) {
for i := 0; i < 16; i++ { for i := 0; i < 16; i++ {
if n.Children[i] != nil { if n.Children[i] != nil {
collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) collapsed.Children[i], cached.Children[i], err = h.hash(prefix, append(path, byte(i)), n.Children[i], db, false)
if err != nil { if err != nil {
return original, original, err return original, original, err
} }
@ -150,7 +150,7 @@ func (h *hasher) hashChildren(original node, db *Database) (node, node, error) {
// store hashes the node n and if we have a storage layer specified, it writes // store hashes the node n and if we have a storage layer specified, it writes
// the key/value pair to it and tracks any node->child references as well as any // the key/value pair to it and tracks any node->child references as well as any
// node->external trie references. // node->external trie references.
func (h *hasher) store(n node, db *Database, force bool) (node, error) { func (h *hasher) store(prefix []byte, path []byte, n node, db *Database, force bool) (node, error) {
// Don't store hashes or empty nodes. // Don't store hashes or empty nodes.
if _, isHash := n.(hashNode); n == nil || isHash { if _, isHash := n.(hashNode); n == nil || isHash {
return n, nil return n, nil
@ -175,18 +175,18 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) {
db.lock.Lock() db.lock.Lock()
hash := common.BytesToHash(hash) hash := common.BytesToHash(hash)
db.insert(hash, h.tmp.Bytes()) db.insert(prefix, hash, h.tmp.Bytes())
// Track all direct parent->child node references // Track all direct parent->child node references
switch n := n.(type) { switch n := n.(type) {
case *shortNode: case *shortNode:
if child, ok := n.Val.(hashNode); ok { if child, ok := n.Val.(hashNode); ok {
db.reference(common.BytesToHash(child), hash) db.reference(prefix, common.BytesToHash(child), prefix, hash)
} }
case *fullNode: case *fullNode:
for i := 0; i < 16; i++ { for i := 0; i < 16; i++ {
if child, ok := n.Children[i].(hashNode); ok { if child, ok := n.Children[i].(hashNode); ok {
db.reference(common.BytesToHash(child), hash) db.reference(prefix, common.BytesToHash(child), prefix, hash)
} }
} }
} }
@ -196,13 +196,13 @@ func (h *hasher) store(n node, db *Database, force bool) (node, error) {
if h.onleaf != nil { if h.onleaf != nil {
switch n := n.(type) { switch n := n.(type) {
case *shortNode: case *shortNode:
if child, ok := n.Val.(valueNode); ok { if child, ok := n.Val.(valueNode); ok && child != nil {
h.onleaf(child, hash) h.onleaf(hexToKeybytes(append(path, compactToHex(n.Key)...)), child, hash)
} }
case *fullNode: case *fullNode:
for i := 0; i < 16; i++ { for i := 0; i < 16; i++ {
if child, ok := n.Children[i].(valueNode); ok { if child, ok := n.Children[i].(valueNode); ok && child != nil {
h.onleaf(child, hash) h.onleaf(hexToKeybytes(append(path, byte(i))), child, hash)
} }
} }
} }

72
trie/iterator_test.go
View file

@ -100,28 +100,34 @@ func TestNodeIteratorCoverage(t *testing.T) {
// Create some arbitrary test trie to iterate // Create some arbitrary test trie to iterate
db, trie, _ := makeTestTrie() db, trie, _ := makeTestTrie()
// Gather all the node hashes found by the iterator // Gather all the node keys found by the iterator
hashes := make(map[common.Hash]struct{}) keys := make(map[string]struct{})
for it := trie.NodeIterator(nil); it.Next(true); { for it := trie.NodeIterator(nil); it.Next(true); {
if it.Hash() != (common.Hash{}) { if it.Hash() != (common.Hash{}) {
hashes[it.Hash()] = struct{}{} keys[dbkey(nil, it.Hash())] = struct{}{}
} }
} }
// Cross check the hashes and the database itself // Cross check the hashes and the database itself
for hash := range hashes { for key := range keys {
if _, err := db.Node(hash); err != nil { var hash common.Hash
copy(hash[:], key[len(key)-common.HashLength:])
if _, err := db.Node(nil, hash); err != nil {
t.Errorf("failed to retrieve reported node %x: %v", hash, err) t.Errorf("failed to retrieve reported node %x: %v", hash, err)
} }
} }
for hash, obj := range db.nodes { for key, obj := range db.nodes {
var hash common.Hash
copy(hash[:], key[len(key)-common.HashLength:])
if obj != nil && hash != (common.Hash{}) { if obj != nil && hash != (common.Hash{}) {
if _, ok := hashes[hash]; !ok { if _, ok := keys[key]; !ok {
t.Errorf("state entry not reported %x", hash) t.Errorf("state entry not reported %x", key)
} }
} }
} }
for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() { for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() {
if _, ok := hashes[common.BytesToHash(key)]; !ok { if _, ok := keys[string(key)]; !ok {
t.Errorf("state entry not reported %x", key) t.Errorf("state entry not reported %x", key)
} }
} }
@ -292,19 +298,19 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
diskdb := ethdb.NewMemDatabase() diskdb := ethdb.NewMemDatabase()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
tr, _ := New(common.Hash{}, triedb) tr, _ := New(nil, common.Hash{}, triedb)
for _, val := range testdata1 { for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v)) tr.Update([]byte(val.k), []byte(val.v))
} }
tr.Commit(nil) tr.Commit(nil)
if !memonly { if !memonly {
triedb.Commit(tr.Hash(), true) triedb.Commit(nil, tr.Hash(), true)
} }
wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil) wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
var ( var (
diskKeys [][]byte diskKeys [][]byte
memKeys []common.Hash memKeys []string
) )
if memonly { if memonly {
memKeys = triedb.Nodes() memKeys = triedb.Nodes()
@ -313,12 +319,14 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
} }
for i := 0; i < 20; i++ { for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB. // Create trie that will load all nodes from DB.
tr, _ := New(tr.Hash(), triedb) tr, err := New(nil, tr.Hash(), triedb)
if err != nil {
panic(err)
}
// Remove a random node from the database. It can't be the root node // Remove a random node from the database. It can't be the root node
// because that one is already loaded. // because that one is already loaded.
var ( var (
rkey common.Hash rkey string
rval []byte rval []byte
robj *cachedNode robj *cachedNode
) )
@ -326,9 +334,9 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
if memonly { if memonly {
rkey = memKeys[rand.Intn(len(memKeys))] rkey = memKeys[rand.Intn(len(memKeys))]
} else { } else {
copy(rkey[:], diskKeys[rand.Intn(len(diskKeys))]) rkey = string(diskKeys[rand.Intn(len(diskKeys))])
} }
if rkey != tr.Hash() { if rkey != dbkey(nil, tr.Hash()) {
break break
} }
} }
@ -336,15 +344,15 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
robj = triedb.nodes[rkey] robj = triedb.nodes[rkey]
delete(triedb.nodes, rkey) delete(triedb.nodes, rkey)
} else { } else {
rval, _ = diskdb.Get(rkey[:]) rval, _ = diskdb.Get([]byte(rkey))
diskdb.Delete(rkey[:]) diskdb.Delete([]byte(rkey))
} }
// Iterate until the error is hit. // Iterate until the error is hit.
seen := make(map[string]bool) seen := make(map[string]bool)
it := tr.NodeIterator(nil) it := tr.NodeIterator(nil)
checkIteratorNoDups(t, it, seen) checkIteratorNoDups(t, it, seen)
missing, ok := it.Error().(*MissingNodeError) missing, ok := it.Error().(*MissingNodeError)
if !ok || missing.NodeHash != rkey { if !ok || !bytes.Equal(missing.NodeHash.Bytes(), []byte(rkey)[len(rkey)-common.HashLength:]) {
t.Fatal("didn't hit missing node, got", it.Error()) t.Fatal("didn't hit missing node, got", it.Error())
} }
@ -352,7 +360,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
if memonly { if memonly {
triedb.nodes[rkey] = robj triedb.nodes[rkey] = robj
} else { } else {
diskdb.Put(rkey[:], rval) diskdb.Put([]byte(rkey), rval)
} }
checkIteratorNoDups(t, it, seen) checkIteratorNoDups(t, it, seen)
if it.Error() != nil { if it.Error() != nil {
@ -379,41 +387,41 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
diskdb := ethdb.NewMemDatabase() diskdb := ethdb.NewMemDatabase()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
ctr, _ := New(common.Hash{}, triedb) ctr, _ := New(nil, common.Hash{}, triedb)
for _, val := range testdata1 { for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v)) ctr.Update([]byte(val.k), []byte(val.v))
} }
root, _ := ctr.Commit(nil) root, _ := ctr.Commit(nil)
if !memonly { if !memonly {
triedb.Commit(root, true) triedb.Commit(nil, root, true)
} }
barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e") barNodeKey := string(common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e").Bytes())
var ( var (
barNodeBlob []byte barNodeBlob []byte
barNodeObj *cachedNode barNodeObj *cachedNode
) )
if memonly { if memonly {
barNodeObj = triedb.nodes[barNodeHash] barNodeObj = triedb.nodes[barNodeKey]
delete(triedb.nodes, barNodeHash) delete(triedb.nodes, barNodeKey)
} else { } else {
barNodeBlob, _ = diskdb.Get(barNodeHash[:]) barNodeBlob, _ = diskdb.Get([]byte(barNodeKey))
diskdb.Delete(barNodeHash[:]) diskdb.Delete([]byte(barNodeKey))
} }
// Create a new iterator that seeks to "bars". Seeking can't proceed because // Create a new iterator that seeks to "bars". Seeking can't proceed because
// the node is missing. // the node is missing.
tr, _ := New(root, triedb) tr, _ := New(nil, root, triedb)
it := tr.NodeIterator([]byte("bars")) it := tr.NodeIterator([]byte("bars"))
missing, ok := it.Error().(*MissingNodeError) missing, ok := it.Error().(*MissingNodeError)
if !ok { if !ok {
t.Fatal("want MissingNodeError, got", it.Error()) t.Fatal("want MissingNodeError, got", it.Error())
} else if missing.NodeHash != barNodeHash { } else if !bytes.Equal(missing.NodeHash.Bytes(), []byte(barNodeKey)[len(barNodeKey)-common.HashLength:]) {
t.Fatal("wrong node missing") t.Fatal("wrong node missing")
} }
// Reinsert the missing node. // Reinsert the missing node.
if memonly { if memonly {
triedb.nodes[barNodeHash] = barNodeObj triedb.nodes[barNodeKey] = barNodeObj
} else { } else {
diskdb.Put(barNodeHash[:], barNodeBlob) diskdb.Put([]byte(barNodeKey), barNodeBlob)
} }
// Check that iteration produces the right set of values. // Check that iteration produces the right set of values.
if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil { if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {

4
trie/proof.go
View file

@ -69,8 +69,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
for i, n := range nodes { for i, n := range nodes {
// Don't bother checking for errors here since hasher panics // Don't bother checking for errors here since hasher panics
// if encoding doesn't work and we're not writing to any database. // if encoding doesn't work and we're not writing to any database.
n, _, _ = hasher.hashChildren(n, nil) n, _, _ = hasher.hashChildren(t.prefix, nil, n, nil)
hn, _ := hasher.store(n, nil, false) hn, _ := hasher.store(t.prefix, nil, n, nil, false)
if hash, ok := hn.(hashNode); ok || i == 0 { if hash, ok := hn.(hashNode); ok || i == 0 {
// If the node's database encoding is a hash (or is the // If the node's database encoding is a hash (or is the
// root node), it becomes a proof element. // root node), it becomes a proof element.

4
trie/secure_trie.go
View file

@ -51,11 +51,11 @@ type SecureTrie struct {
// Loaded nodes are kept around until their 'cache generation' expires. // Loaded nodes are kept around until their 'cache generation' expires.
// A new cache generation is created by each call to Commit. // A new cache generation is created by each call to Commit.
// cachelimit sets the number of past cache generations to keep. // cachelimit sets the number of past cache generations to keep.
func NewSecure(root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) { func NewSecure(prefix []byte, root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) {
if db == nil { if db == nil {
panic("trie.NewSecure called without a database") panic("trie.NewSecure called without a database")
} }
trie, err := New(root, db) trie, err := New(prefix, root, db)
if err != nil { if err != nil {
return nil, err return nil, err
} }

4
trie/secure_trie_test.go
View file

@ -28,7 +28,7 @@ import (
) )
func newEmptySecure() *SecureTrie { func newEmptySecure() *SecureTrie {
trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0) trie, _ := NewSecure(nil, common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0)
return trie return trie
} }
@ -37,7 +37,7 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie // Create an empty trie
triedb := NewDatabase(ethdb.NewMemDatabase()) triedb := NewDatabase(ethdb.NewMemDatabase())
trie, _ := NewSecure(common.Hash{}, triedb, 0) trie, _ := NewSecure(nil, common.Hash{}, triedb, 0)
// Fill it with some arbitrary data // Fill it with some arbitrary data
content := make(map[string][]byte) content := make(map[string][]byte)

2
trie/sync.go
View file

@ -280,7 +280,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
// Notify any external watcher of a new key/value node // Notify any external watcher of a new key/value node
if req.callback != nil { if req.callback != nil {
if node, ok := (child.node).(valueNode); ok { if node, ok := (child.node).(valueNode); ok {
if err := req.callback(node, req.hash); err != nil { if err := req.callback(nil, node, req.hash); err != nil {
return nil, err return nil, err
} }
} }

22
trie/sync_test.go
View file

@ -28,7 +28,7 @@ import (
func makeTestTrie() (*Database, *Trie, map[string][]byte) { func makeTestTrie() (*Database, *Trie, map[string][]byte) {
// Create an empty trie // Create an empty trie
triedb := NewDatabase(ethdb.NewMemDatabase()) triedb := NewDatabase(ethdb.NewMemDatabase())
trie, _ := New(common.Hash{}, triedb) trie, _ := New(nil, common.Hash{}, triedb)
// Fill it with some arbitrary data // Fill it with some arbitrary data
content := make(map[string][]byte) content := make(map[string][]byte)
@ -59,7 +59,7 @@ func makeTestTrie() (*Database, *Trie, map[string][]byte) {
// content map. // content map.
func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
// Check root availability and trie contents // Check root availability and trie contents
trie, err := New(common.BytesToHash(root), db) trie, err := New(nil, common.BytesToHash(root), db)
if err != nil { if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err) t.Fatalf("failed to create trie at %x: %v", root, err)
} }
@ -76,7 +76,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri
// checkTrieConsistency checks that all nodes in a trie are indeed present. // checkTrieConsistency checks that all nodes in a trie are indeed present.
func checkTrieConsistency(db *Database, root common.Hash) error { func checkTrieConsistency(db *Database, root common.Hash) error {
// Create and iterate a trie rooted in a subnode // Create and iterate a trie rooted in a subnode
trie, err := New(root, db) trie, err := New(nil, root, db)
if err != nil { if err != nil {
return nil // Consider a non existent state consistent return nil // Consider a non existent state consistent
} }
@ -90,8 +90,8 @@ func checkTrieConsistency(db *Database, root common.Hash) error {
func TestEmptyTrieSync(t *testing.T) { func TestEmptyTrieSync(t *testing.T) {
dbA := NewDatabase(ethdb.NewMemDatabase()) dbA := NewDatabase(ethdb.NewMemDatabase())
dbB := NewDatabase(ethdb.NewMemDatabase()) dbB := NewDatabase(ethdb.NewMemDatabase())
emptyA, _ := New(common.Hash{}, dbA) emptyA, _ := New(nil, common.Hash{}, dbA)
emptyB, _ := New(emptyRoot, dbB) emptyB, _ := New(nil, emptyRoot, dbB)
for i, trie := range []*Trie{emptyA, emptyB} { for i, trie := range []*Trie{emptyA, emptyB} {
if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 { if req := NewTrieSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 {
@ -118,7 +118,7 @@ func testIterativeTrieSync(t *testing.T, batch int) {
for len(queue) > 0 { for len(queue) > 0 {
results := make([]SyncResult, len(queue)) results := make([]SyncResult, len(queue))
for i, hash := range queue { for i, hash := range queue {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }
@ -152,7 +152,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
// Sync only half of the scheduled nodes // Sync only half of the scheduled nodes
results := make([]SyncResult, len(queue)/2+1) results := make([]SyncResult, len(queue)/2+1)
for i, hash := range queue[:len(results)] { for i, hash := range queue[:len(results)] {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }
@ -193,7 +193,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
// Fetch all the queued nodes in a random order // Fetch all the queued nodes in a random order
results := make([]SyncResult, 0, len(queue)) results := make([]SyncResult, 0, len(queue))
for hash := range queue { for hash := range queue {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }
@ -234,7 +234,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
// Sync only half of the scheduled nodes, even those in random order // Sync only half of the scheduled nodes, even those in random order
results := make([]SyncResult, 0, len(queue)/2+1) results := make([]SyncResult, 0, len(queue)/2+1)
for hash := range queue { for hash := range queue {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }
@ -279,7 +279,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
for len(queue) > 0 { for len(queue) > 0 {
results := make([]SyncResult, len(queue)) results := make([]SyncResult, len(queue))
for i, hash := range queue { for i, hash := range queue {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }
@ -319,7 +319,7 @@ func TestIncompleteTrieSync(t *testing.T) {
// Fetch a batch of trie nodes // Fetch a batch of trie nodes
results := make([]SyncResult, len(queue)) results := make([]SyncResult, len(queue))
for i, hash := range queue { for i, hash := range queue {
data, err := srcDb.Node(hash) data, err := srcDb.Node(nil /*TODO*/, hash)
if err != nil { if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err) t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
} }

11
trie/trie.go
View file

@ -57,7 +57,7 @@ func CacheUnloads() int64 {
// LeafCallback is a callback type invoked when a trie operation reaches a leaf // LeafCallback is a callback type invoked when a trie operation reaches a leaf
// node. It's used by state sync and commit to allow handling external references // node. It's used by state sync and commit to allow handling external references
// between account and storage tries. // between account and storage tries.
type LeafCallback func(leaf []byte, parent common.Hash) error type LeafCallback func(path []byte, leaf []byte, parent common.Hash) error
// Trie is a Merkle Patricia Trie. // Trie is a Merkle Patricia Trie.
// The zero value is an empty trie with no database. // The zero value is an empty trie with no database.
@ -67,6 +67,7 @@ type LeafCallback func(leaf []byte, parent common.Hash) error
type Trie struct { type Trie struct {
db *Database db *Database
root node root node
prefix []byte
originalRoot common.Hash originalRoot common.Hash
// Cache generation values. // Cache generation values.
@ -93,12 +94,13 @@ func (t *Trie) newFlag() nodeFlag {
// trie is initially empty and does not require a database. Otherwise, // trie is initially empty and does not require a database. Otherwise,
// New will panic if db is nil and returns a MissingNodeError if root does // New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand. // not exist in the database. Accessing the trie loads nodes from db on demand.
func New(root common.Hash, db *Database) (*Trie, error) { func New(prefix []byte, root common.Hash, db *Database) (*Trie, error) {
if db == nil { if db == nil {
panic("trie.New called without a database") panic("trie.New called without a database")
} }
trie := &Trie{ trie := &Trie{
db: db, db: db,
prefix: prefix,
originalRoot: root, originalRoot: root,
} }
if (root != common.Hash{}) && root != emptyRoot { if (root != common.Hash{}) && root != emptyRoot {
@ -434,7 +436,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
hash := common.BytesToHash(n) hash := common.BytesToHash(n)
enc, err := t.db.Node(hash) enc, err := t.db.Node(t.prefix, hash)
if err != nil || enc == nil { if err != nil || enc == nil {
return nil, &MissingNodeError{NodeHash: hash, Path: prefix} return nil, &MissingNodeError{NodeHash: hash, Path: prefix}
} }
@ -474,5 +476,6 @@ func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) {
} }
h := newHasher(t.cachegen, t.cachelimit, onleaf) h := newHasher(t.cachegen, t.cachelimit, onleaf)
defer returnHasherToPool(h) defer returnHasherToPool(h)
return h.hash(t.root, db, true)
return h.hash(t.prefix, nil, t.root, db, true)
} }

44
trie/trie_test.go
View file

@ -43,7 +43,7 @@ func init() {
// Used for testing // Used for testing
func newEmpty() *Trie { func newEmpty() *Trie {
trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase())) trie, _ := New(nil, common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
return trie return trie
} }
@ -67,7 +67,7 @@ func TestNull(t *testing.T) {
} }
func TestMissingRoot(t *testing.T) { func TestMissingRoot(t *testing.T) {
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase())) trie, err := New(nil, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase()))
if trie != nil { if trie != nil {
t.Error("New returned non-nil trie for invalid root") t.Error("New returned non-nil trie for invalid root")
} }
@ -83,35 +83,35 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb := ethdb.NewMemDatabase() diskdb := ethdb.NewMemDatabase()
triedb := NewDatabase(diskdb) triedb := NewDatabase(diskdb)
trie, _ := New(common.Hash{}, triedb) trie, _ := New(nil, common.Hash{}, triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _ := trie.Commit(nil) root, _ := trie.Commit(nil)
if !memonly { if !memonly {
triedb.Commit(root, true) triedb.Commit(nil, root, true)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err := trie.TryGet([]byte("120000")) _, err := trie.TryGet([]byte("120000"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err = trie.TryGet([]byte("120099")) _, err = trie.TryGet([]byte("120099"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err = trie.TryGet([]byte("123456")) _, err = trie.TryGet([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
err = trie.TryDelete([]byte("123456")) err = trie.TryDelete([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
@ -119,32 +119,32 @@ func testMissingNode(t *testing.T, memonly bool) {
hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
if memonly { if memonly {
delete(triedb.nodes, hash) delete(triedb.nodes, string(hash[:]))
} else { } else {
diskdb.Delete(hash[:]) diskdb.Delete(hash[:])
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err = trie.TryGet([]byte("120000")) _, err = trie.TryGet([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err = trie.TryGet([]byte("120099")) _, err = trie.TryGet([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
_, err = trie.TryGet([]byte("123456")) _, err = trie.TryGet([]byte("123456"))
if err != nil { if err != nil {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
} }
trie, _ = New(root, triedb) trie, _ = New(nil, root, triedb)
err = trie.TryDelete([]byte("123456")) err = trie.TryDelete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok { if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err) t.Errorf("Wrong error: %v", err)
@ -272,7 +272,7 @@ func TestReplication(t *testing.T) {
} }
// create a new trie on top of the database and check that lookups work. // create a new trie on top of the database and check that lookups work.
trie2, err := New(exp, trie.db) trie2, err := New(nil, exp, trie.db)
if err != nil { if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err) t.Fatalf("can't recreate trie at %x: %v", exp, err)
} }
@ -337,13 +337,13 @@ func TestCacheUnload(t *testing.T) {
updateString(trie, key2, "this is the branch of key2.") updateString(trie, key2, "this is the branch of key2.")
root, _ := trie.Commit(nil) root, _ := trie.Commit(nil)
trie.db.Commit(root, true) trie.db.Commit(nil, root, true)
// Commit the trie repeatedly and access key1. // Commit the trie repeatedly and access key1.
// The branch containing it is loaded from DB exactly two times: // The branch containing it is loaded from DB exactly two times:
// in the 0th and 6th iteration. // in the 0th and 6th iteration.
db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)} db := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)}
trie, _ = New(root, NewDatabase(db)) trie, _ = New(nil, root, NewDatabase(db))
trie.SetCacheLimit(5) trie.SetCacheLimit(5)
for i := 0; i < 12; i++ { for i := 0; i < 12; i++ {
getString(trie, key1) getString(trie, key1)
@ -413,7 +413,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
func runRandTest(rt randTest) bool { func runRandTest(rt randTest) bool {
triedb := NewDatabase(ethdb.NewMemDatabase()) triedb := NewDatabase(ethdb.NewMemDatabase())
tr, _ := New(common.Hash{}, triedb) tr, _ := New(nil, common.Hash{}, triedb)
values := make(map[string]string) // tracks content of the trie values := make(map[string]string) // tracks content of the trie
for i, step := range rt { for i, step := range rt {
@ -440,14 +440,14 @@ func runRandTest(rt randTest) bool {
rt[i].err = err rt[i].err = err
return false return false
} }
newtr, err := New(hash, triedb) newtr, err := New(nil, hash, triedb)
if err != nil { if err != nil {
rt[i].err = err rt[i].err = err
return false return false
} }
tr = newtr tr = newtr
case opItercheckhash: case opItercheckhash:
checktr, _ := New(common.Hash{}, triedb) checktr, _ := New(nil, common.Hash{}, triedb)
it := NewIterator(tr.NodeIterator(nil)) it := NewIterator(tr.NodeIterator(nil))
for it.Next() { for it.Next() {
checktr.Update(it.Key, it.Value) checktr.Update(it.Key, it.Value)
@ -520,7 +520,7 @@ func benchGet(b *testing.B, commit bool) {
trie := new(Trie) trie := new(Trie)
if commit { if commit {
_, tmpdb := tempDB() _, tmpdb := tempDB()
trie, _ = New(common.Hash{}, tmpdb) trie, _ = New(nil, common.Hash{}, tmpdb)
} }
k := make([]byte, 32) k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ { for i := 0; i < benchElemCount; i++ {