Merge 0b12010d84 into 1b73c79234

cmd/geth/chaincmd.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
@@ -118,7 +119,10 @@ func importChain(ctx *cli.Context) error {
 	}
 	fmt.Println(stats)
 	fmt.Printf("Trie cache misses:   %d\n", trie.CacheMisses())
-	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
+	fmt.Printf("Trie cache unloads:  %d\n", trie.CacheUnloads())
+	fmt.Printf("Direct cache reads:  %d\n", trie.DirectCacheReads())
+	fmt.Printf("Direct cache writes: %d\n", trie.DirectCacheWrites())
+	fmt.Printf("Direct cache misses: %d\n\n", trie.DirectCacheMisses())
 
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)
@@ -271,6 +275,49 @@ func dump(ctx *cli.Context) error {
 	return nil
 }
 
+var accountCacheKeyPrefix = []byte("accounthashcache:")
+
+type cachedAccount struct {
+	state.Account
+	BlockNum  uint64
+	BlockHash common.Hash
+}
+
+func buildCache(ctx *cli.Context) error {
+	stack := makeFullNode(ctx)
+	chain, chainDb := utils.MakeChain(ctx, stack)
+	defer chainDb.Close()
+
+	num, _ := strconv.Atoi(ctx.Args()[0])
+	blockNum := uint64(num)
+	block := chain.GetBlockByNumber(blockNum)
+	blockHash := block.Hash()
+	t, err := trie.New(block.Root(), chainDb, 0)
+	if err != nil {
+		utils.Fatalf("Could not open trie: %v", err)
+	}
+	st := trie.NewSecure(t, chainDb)
+	iter := st.Iterator()
+	i := 0
+	for iter.Next() {
+		var data state.Account
+		if err := rlp.DecodeBytes(iter.Value, &data); err != nil {
+			utils.Fatalf("can't decode object at %x: %v", iter.Key[:], err)
+		}
+
+		enc, _ := rlp.EncodeToBytes(cachedAccount{data, blockNum, blockHash})
+		if err := chainDb.Put(append(accountCacheKeyPrefix, iter.Key[:]...), enc); err != nil {
+			utils.Fatalf("Could not write to DB: %v", err)
+		}
+
+		i += 1
+		if i%10000 == 0 {
+			fmt.Printf("Processed %d accounts, at %v\n", i, common.ToHex(iter.Key))
+		}
+	}
+	return nil
+}
+
 // hashish returns true for strings that look like hashes.
 func hashish(x string) bool {
 	_, err := strconv.Atoi(x)

core/blockchain.go

@@ -269,7 +269,7 @@ func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x…]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), self.chainDb, 0); err != nil {
+	if _, err := trie.New(block.Root(), self.chainDb, 0); err != nil {
 		return err
 	}
 	// If all checks out, manually set the head block
@@ -545,6 +545,11 @@ func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block {
 	return self.GetBlock(hash, number)
 }
 
+// IsCanonChainBlock checks whether the given block is in the current canonical chain.
+func (self *BlockChain) IsCanonChainBlock(number uint64, hash common.Hash) bool {
+	return GetCanonicalHash(self.chainDb, number) == hash
+}
+
 // [deprecated by eth/62]
 // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.
 func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) {
@@ -917,6 +922,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			reportBlock(block, err)
 			return i, err
 		}
+		self.stateCache.SetBlockContext(block.Hash(), block.NumberU64(), self)
 		// Process block using the parent state as reference point.
 		receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.config.VmConfig)
 		if err != nil {

core/chain_makers.go

@@ -105,7 +105,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
 	if b.gasPool == nil {
 		b.SetCoinbase(common.Address{})
 	}
-	b.statedb.StartRecord(tx.Hash(), common.Hash{}, len(b.txs))
+	b.statedb.SetTxContext(tx.Hash(), len(b.txs))
 	receipt, _, _, err := ApplyTransaction(MakeChainConfig(), nil, b.gasPool, b.statedb, b.header, tx, b.header.GasUsed, vm.Config{})
 	if err != nil {
 		panic(err)

core/state/dump.go

@@ -44,9 +44,9 @@ func (self *StateDB) RawDump() Dump {
 		Accounts: make(map[string]DumpAccount),
 	}
 
-	it := self.trie.Iterator()
+	it := self.storage.Iterator()
 	for it.Next() {
-		addr := self.trie.GetKey(it.Key)
+		addr := self.storage.GetKey(it.Key)
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
@@ -61,9 +61,9 @@ func (self *StateDB) RawDump() Dump {
 			Code:     common.Bytes2Hex(obj.Code(self.db)),
 			Storage:  make(map[string]string),
 		}
-		storageIt := obj.getTrie(self.db).Iterator()
+		storageIt := obj.getStorage(self.db).Iterator()
 		for storageIt.Next() {
-			account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+			account.Storage[common.Bytes2Hex(self.storage.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
 		}
 		dump.Accounts[common.Bytes2Hex(addr)] = account
 	}

core/state/iterator.go

@@ -76,7 +76,7 @@ func (it *NodeIterator) step() error {
 	}
 	// Initialize the iterator if we've just started
 	if it.stateIt == nil {
-		it.stateIt = it.state.trie.NodeIterator()
+		it.stateIt = trie.NewNodeIterator(it.state.trie)
 	}
 	// If we had data nodes previously, we surely have at least state nodes
 	if it.dataIt != nil {
@@ -115,7 +115,7 @@ func (it *NodeIterator) step() error {
 	if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob), &account); err != nil {
 		return err
 	}
-	dataTrie, err := trie.New(account.Root, it.state.db)
+	dataTrie, err := trie.New(account.Root, it.state.db, 0)
 	if err != nil {
 		return err
 	}

core/state/iterator_test.go

@@ -47,7 +47,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
 		}
 	}
 	for _, key := range db.(*ethdb.MemDatabase).Keys() {
-		if bytes.HasPrefix(key, []byte("secure-key-")) {
+		if bytes.HasPrefix(key, []byte("secure-key-")) || bytes.HasPrefix(key, DirectCachePrefix) {
 			continue
 		}
 		if _, ok := hashes[common.BytesToHash(key)]; !ok {

core/state/state_object.go

@@ -76,7 +76,8 @@ type StateObject struct {
 	dbErr error
 
 	// Write caches.
-	trie *trie.SecureTrie // storage trie, which becomes non-nil on first access
+	trie *trie.Trie // storage trie, which becomes non-nil on first access
+	storage *trie.SecureTrie // Interface to storage
 	code Code             // contract bytecode, which gets set when code is loaded
 
 	cachedStorage Storage // Storage entry cache to avoid duplicate reads
@@ -134,16 +135,17 @@ func (self *StateObject) markSuicided() {
 	}
 }
 
-func (c *StateObject) getTrie(db trie.Database) *trie.SecureTrie {
+func (c *StateObject) getStorage(db trie.Database) *trie.SecureTrie {
 	if c.trie == nil {
 		var err error
-		c.trie, err = trie.NewSecure(c.data.Root, db, 0)
+		c.trie, err = trie.New(c.data.Root, db, 0)
 		if err != nil {
-			c.trie, _ = trie.NewSecure(common.Hash{}, db, 0)
+			c.trie, _ = trie.New(common.Hash{}, nil, 0)
 			c.setError(fmt.Errorf("can't create storage trie: %v", err))
 		}
+		c.storage = trie.NewSecure(c.trie, db)
 	}
-	return c.trie
+	return c.storage
 }
 
 // GetState returns a value in account storage.
@@ -153,7 +155,7 @@ func (self *StateObject) GetState(db trie.Database, key common.Hash) common.Hash
 		return value
 	}
 	// Load from DB in case it is missing.
-	if enc := self.getTrie(db).Get(key[:]); len(enc) > 0 {
+	if enc := self.getStorage(db).Get(key[:]); len(enc) > 0 {
 		_, content, _, err := rlp.Split(enc)
 		if err != nil {
 			self.setError(err)
@@ -188,7 +190,7 @@ func (self *StateObject) setState(key, value common.Hash) {
 
 // updateTrie writes cached storage modifications into the object's storage trie.
 func (self *StateObject) updateTrie(db trie.Database) {
-	tr := self.getTrie(db)
+	tr := self.getStorage(db)
 	for key, value := range self.dirtyStorage {
 		delete(self.dirtyStorage, key)
 		if (value == common.Hash{}) {
@@ -214,7 +216,7 @@ func (self *StateObject) CommitTrie(db trie.Database, dbw trie.DatabaseWriter) e
 	if self.dbErr != nil {
 		return self.dbErr
 	}
-	root, err := self.trie.CommitTo(dbw)
+	root, err := self.storage.CommitTo(dbw)
 	if err == nil {
 		self.data.Root = root
 	}
@@ -265,6 +267,7 @@ func (c *StateObject) ReturnGas(gas, price *big.Int) {}
 func (self *StateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *StateObject {
 	stateObject := newObject(db, self.address, self.data, onDirty)
 	stateObject.trie = self.trie
+	stateObject.storage = self.storage
 	stateObject.code = self.code
 	stateObject.dirtyStorage = self.dirtyStorage.Copy()
 	stateObject.cachedStorage = self.dirtyStorage.Copy()
@@ -360,10 +363,10 @@ func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) {
 		cb(h, value)
 	}
 
-	it := self.getTrie(self.db.db).Iterator()
+	it := self.getStorage(self.db.db).Iterator()
 	for it.Next() {
 		// ignore cached values
-		key := common.BytesToHash(self.trie.GetKey(it.Key))
+		key := common.BytesToHash(self.storage.GetKey(it.Key))
 		if _, ok := self.cachedStorage[key]; !ok {
 			cb(key, common.BytesToHash(it.Value))
 		}

core/state/statedb.go

@@ -41,6 +41,8 @@ var StartingNonce uint64
 // Trie cache generation limit after which to evic trie nodes from memory.
 var MaxTrieCacheGen = uint16(120)
 
+var DirectCachePrefix = []byte("accounthashcache:")
+
 const (
 	// Number of past tries to keep. This value is chosen such that
 	// reasonable chain reorg depths will hit an existing trie.
@@ -62,8 +64,10 @@ type revision struct {
 // * Accounts
 type StateDB struct {
 	db            ethdb.Database
-	trie          *trie.SecureTrie
+	trie          *trie.Trie
-	pastTries     []*trie.SecureTrie
+	storage       *trie.SecureTrie
+	cacheComplete bool  // True if we know that the directcache is complete
+	pastTries     []*trie.Trie
 	codeSizeCache *lru.Cache
 
 	// This map holds 'live' objects, which will get modified while processing a state transition.
@@ -89,12 +93,13 @@ type StateDB struct {
 
 // Create a new state from a given trie
 func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
-	tr, err := trie.NewSecure(root, db, MaxTrieCacheGen)
+	tr, err := trie.New(root, db, MaxTrieCacheGen)
 	if err != nil {
 		return nil, err
 	}
+
 	csc, _ := lru.New(codeSizeCacheSize)
-	return &StateDB{
+	ret := &StateDB{
 		db:                db,
 		trie:              tr,
 		codeSizeCache:     csc,
@@ -102,7 +107,9 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		refund:            new(big.Int),
 		logs:              make(map[common.Hash]vm.Logs),
-	}, nil
+	}
+	ret.SetBlockContext(common.Hash{}, 0, nil)
+	return ret, nil
 }
 
 // New creates a new statedb by reusing any journalled tries to avoid costly
@@ -115,7 +122,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &StateDB{
+	ret := &StateDB{
 		db:                self.db,
 		trie:              tr,
 		codeSizeCache:     self.codeSizeCache,
@@ -123,7 +130,9 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		refund:            new(big.Int),
 		logs:              make(map[common.Hash]vm.Logs),
-	}, nil
+	}
+	ret.SetBlockContext(common.Hash{}, 0, nil)
+	return ret, nil
 }
 
 // Reset clears out all emphemeral state objects from the state db, but keeps
@@ -145,23 +154,25 @@ func (self *StateDB) Reset(root common.Hash) error {
 	self.logs = make(map[common.Hash]vm.Logs)
 	self.logSize = 0
 	self.clearJournalAndRefund()
+	self.SetBlockContext(common.Hash{}, 0, nil)
+	self.SetTxContext(common.Hash{}, 0)
 
 	return nil
 }
 
 // openTrie creates a trie. It uses an existing trie if one is available
 // from the journal if available.
-func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) {
+func (self *StateDB) openTrie(root common.Hash) (*trie.Trie, error) {
 	for i := len(self.pastTries) - 1; i >= 0; i-- {
 		if self.pastTries[i].Hash() == root {
 			tr := *self.pastTries[i]
 			return &tr, nil
 		}
 	}
-	return trie.NewSecure(root, self.db, MaxTrieCacheGen)
+	return trie.New(root, self.db, MaxTrieCacheGen)
 }
 
-func (self *StateDB) pushTrie(t *trie.SecureTrie) {
+func (self *StateDB) pushTrie(t *trie.Trie) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
 
@@ -173,10 +184,26 @@ func (self *StateDB) pushTrie(t *trie.SecureTrie) {
 	}
 }
 
-func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) {
+func (self *StateDB) SetBlockContext(blockHash common.Hash, blockNum uint64, validator trie.CacheValidator) {
-	self.thash = thash
+	self.bhash = blockHash
-	self.bhash = bhash
+	if validator == nil {
-	self.txIndex = ti
+		validator = &trie.NullCacheValidator{}
+	}
+
+	// Check if cache population has completed
+	if !self.cacheComplete {
+		if _, status := trie.GetMigrationState(DirectCachePrefix, self.db); status == trie.Complete {
+			self.cacheComplete = true
+		}
+	}
+
+	storage := trie.NewDirectCache(self.trie, self.db, DirectCachePrefix, blockNum, blockHash, validator, self.cacheComplete)
+	self.storage = trie.NewSecure(storage, self.db)
+}
+
+func (self *StateDB) SetTxContext(txHash common.Hash, txIndex int) {
+	self.thash = txHash
+	self.txIndex = txIndex
 }
 
 func (self *StateDB) AddLog(log *vm.Log) {
@@ -356,14 +383,14 @@ func (self *StateDB) updateStateObject(stateObject *StateObject) {
 	if err != nil {
 		panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
 	}
-	self.trie.Update(addr[:], data)
+	self.storage.Update(addr[:], data)
 }
 
 // deleteStateObject removes the given object from the state trie.
 func (self *StateDB) deleteStateObject(stateObject *StateObject) {
 	stateObject.deleted = true
 	addr := stateObject.Address()
-	self.trie.Delete(addr[:])
+	self.storage.Delete(addr[:])
 }
 
 // Retrieve a state object given my the address. Returns nil if not found.
@@ -377,7 +404,7 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje
 	}
 
 	// Load the object from the database.
-	enc := self.trie.Get(addr[:])
+	enc := self.storage.Get(addr[:])
 	if len(enc) == 0 {
 		return nil
 	}
@@ -457,6 +484,7 @@ func (self *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                self.db,
 		trie:              self.trie,
+		storage:           self.storage,
 		pastTries:         self.pastTries,
 		codeSizeCache:     self.codeSizeCache,
 		stateObjects:      make(map[common.Address]*StateObject, len(self.stateObjectsDirty)),
@@ -602,7 +630,7 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error)
 		delete(s.stateObjectsDirty, addr)
 	}
 	// Write trie changes.
-	root, err = s.trie.CommitTo(dbw)
+	root, err = s.storage.CommitTo(dbw)
 	if err == nil {
 		s.pushTrie(s.trie)
 	}

core/state/sync.go

@@ -32,10 +32,11 @@ import (
 type StateSync trie.TrieSync
 
 // NewStateSync create a new state trie download scheduler.
-func NewStateSync(root common.Hash, database ethdb.Database) *StateSync {
+func NewStateSync(number uint64, hash common.Hash, root common.Hash, database ethdb.Database) *StateSync {
 	var syncer *trie.TrieSync
 
-	callback := func(leaf []byte, parent common.Hash) error {
+	callback := func(keys [][]byte, leaf []byte, parent common.Hash) error {
+		// Try to decode any leaf nodes as accounts
 		var obj struct {
 			Nonce    uint64
 			Balance  *big.Int
@@ -45,12 +46,24 @@ func NewStateSync(root common.Hash, database ethdb.Database) *StateSync {
 		if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil {
 			return err
 		}
-		syncer.AddSubTrie(obj.Root, 64, parent, nil)
+		// Populate the direct account caches in the database
+		if err := trie.DirectCacheTransaction(func() error {
+			for _, key := range keys {
+				if err := trie.WriteDirectCache(DirectCachePrefix, key, leaf, number, hash, database); err != nil {
+					return err
+				}
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+		// Schedule downloading all the dependencies of the account
+		syncer.AddSubTrie(obj.Root, 64, parent, nil, nil)
 		syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent)
 
 		return nil
 	}
-	syncer = trie.NewTrieSync(root, database, callback)
+	syncer = trie.NewTrieSync(root, database, callback, DirectCachePrefix)
 	return (*StateSync)(syncer)
 }
 

core/state/sync_test.go

@@ -47,8 +47,8 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
 		obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
 		acc := &testAccount{address: common.BytesToAddress([]byte{i})}
 
-		obj.AddBalance(big.NewInt(int64(11 * i)))
+		obj.AddBalance(big.NewInt(11 * int64(i)))
-		acc.balance = big.NewInt(int64(11 * i))
+		acc.balance = big.NewInt(11 * int64(i))
 
 		obj.SetNonce(uint64(42 * i))
 		acc.nonce = uint64(42 * i)
@@ -110,7 +110,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
 func TestEmptyStateSync(t *testing.T) {
 	empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 	db, _ := ethdb.NewMemDatabase()
-	if req := NewStateSync(empty, db).Missing(1); len(req) != 0 {
+	if req := NewStateSync(0, common.Hash{}, empty, db).Missing(1); len(req) != 0 {
 		t.Errorf("content requested for empty state: %v", req)
 	}
 }
@@ -126,7 +126,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewStateSync(srcRoot, dstDb)
+	sched := NewStateSync(0, common.Hash{}, srcRoot, dstDb)
 
 	queue := append([]common.Hash{}, sched.Missing(batch)...)
 	for len(queue) > 0 {
@@ -155,7 +155,7 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewStateSync(srcRoot, dstDb)
+	sched := NewStateSync(0, common.Hash{}, srcRoot, dstDb)
 
 	queue := append([]common.Hash{}, sched.Missing(0)...)
 	for len(queue) > 0 {
@@ -189,7 +189,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewStateSync(srcRoot, dstDb)
+	sched := NewStateSync(0, common.Hash{}, srcRoot, dstDb)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(batch) {
@@ -226,7 +226,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewStateSync(srcRoot, dstDb)
+	sched := NewStateSync(0, common.Hash{}, srcRoot, dstDb)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(0) {
@@ -268,7 +268,7 @@ func TestIncompleteStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewStateSync(srcRoot, dstDb)
+	sched := NewStateSync(0, common.Hash{}, srcRoot, dstDb)
 
 	added := []common.Hash{}
 	queue := append([]common.Hash{}, sched.Missing(1)...)

core/state_processor.go

@@ -71,7 +71,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	}
 	// Iterate over and process the individual transactions
 	for i, tx := range block.Transactions() {
-		statedb.StartRecord(tx.Hash(), block.Hash(), i)
+		statedb.SetTxContext(tx.Hash(), i)
 		receipt, logs, _, err := ApplyTransaction(p.config, p.bc, gp, statedb, header, tx, totalUsedGas, cfg)
 		if err != nil {
 			return nil, nil, totalUsedGas, err

core/types/derive_sha.go

@@ -31,7 +31,7 @@ type DerivableList interface {
 
 func DeriveSha(list DerivableList) common.Hash {
 	keybuf := new(bytes.Buffer)
-	trie := new(trie.Trie)
+	trie, _ := trie.New(common.Hash{}, nil, 0)
 	for i := 0; i < list.Len(); i++ {
 		keybuf.Reset()
 		rlp.Encode(keybuf, uint(i))

eth/backend.go

@@ -235,6 +235,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 	gpo := gasprice.NewGasPriceOracle(eth.blockchain, chainDb, eth.eventMux, gpoParams)
 	eth.apiBackend = &EthApiBackend{eth, gpo}
 
+	if err := populateDirectCache(chainDb, eth.blockchain); err != nil {
+		return nil, err
+	}
+
 	return eth, nil
 }
 

eth/db_upgrade.go

@@ -26,11 +26,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 var useSequentialKeys = []byte("dbUpgrade_20160530sequentialKeys")
@@ -354,3 +356,33 @@ func addMipmapBloomBins(db ethdb.Database) (err error) {
 	glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart))
 	return nil
 }
+
+func populateDirectCache(db ethdb.Database, bc *core.BlockChain) error {
+	blockHash, status := trie.GetMigrationState(state.DirectCachePrefix, db)
+	if status == trie.NotStarted {
+		blockHash = core.GetHeadBlockHash(db)
+		status = trie.Running
+		trie.SetMigrationState(state.DirectCachePrefix, blockHash, status, db)
+	}
+	blockNum := core.GetBlockNumber(db, blockHash)
+
+	if status == trie.Running {
+		tr, err := trie.New(core.GetBlock(db, blockHash, blockNum).Root(), db, 0)
+		if err != nil {
+			return err
+		}
+		dc := trie.NewDirectCache(tr, db, state.DirectCachePrefix, blockNum, blockHash, bc, false)
+		go func() {
+			for core.GetBlockNumber(db, core.GetHeadBlockHash(db)) < blockNum + 8 {
+				time.Sleep(10 * time.Second)
+			}
+			glog.Infof("Beginning direct cache upgrade at block %v", blockNum)
+
+			if err := dc.Populate(); err != nil {
+				glog.Errorf("Direct cache upgrade failed: %v", err)
+			}
+			trie.SetMigrationState(state.DirectCachePrefix, blockHash, trie.Complete, db)
+		}()
+	}
+	return nil
+}

eth/downloader/downloader_test.go

@@ -293,7 +293,7 @@ func (dl *downloadTester) headFastBlock() *types.Block {
 func (dl *downloadTester) commitHeadBlock(hash common.Hash) error {
 	// For now only check that the state trie is correct
 	if block := dl.getBlock(hash); block != nil {
-		_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
+		_, err := trie.New(block.Root(), dl.stateDb, 0)
 		return err
 	}
 	return fmt.Errorf("non existent block: %x", hash[:4])

eth/downloader/queue.go

@@ -399,7 +399,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
 			}
 
 			q.stateSchedLock.Lock()
-			q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase)
+			q.stateScheduler = state.NewStateSync(header.Number.Uint64(), header.Hash(), header.Root, q.stateDatabase)
 			q.stateSchedLock.Unlock()
 		}
 		inserts = append(inserts, header)
@@ -1152,6 +1152,6 @@ func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64, head *types.
 
 	// If long running fast sync, also start up a head stateretrieval immediately
 	if mode == FastSync && pivot > 0 {
-		q.stateScheduler = state.NewStateSync(head.Root, q.stateDatabase)
+		q.stateScheduler = state.NewStateSync(head.Number.Uint64(), head.Hash(), head.Root, q.stateDatabase)
 	}
 }

light/state_test.go

@@ -40,7 +40,7 @@ func (odr *testOdr) Database() ethdb.Database {
 func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
 	switch req := req.(type) {
 	case *TrieRequest:
-		t, _ := trie.New(req.root, odr.sdb)
+		t, _ := trie.New(req.root, odr.sdb, 0)
 		req.proof = t.Prove(req.key)
 	case *NodeDataRequest:
 		req.data, _ = odr.sdb.Get(req.hash[:])

light/trie.go

@@ -25,7 +25,7 @@ import (
 
 // LightTrie is an ODR-capable wrapper around trie.SecureTrie
 type LightTrie struct {
-	trie         *trie.SecureTrie
+	data         *trie.SecureTrie
 	originalRoot common.Hash
 	odr          OdrBackend
 	db           ethdb.Database
@@ -74,15 +74,26 @@ func (t *LightTrie) do(ctx context.Context, fallbackKey []byte, fn func() error)
 	return err
 }
 
+func (t *LightTrie) getMap() (ret *trie.SecureTrie, err error) {
+	if t.data == nil {
+		var tr trie.PersistentMap
+		tr, err = trie.New(t.originalRoot, t.db, 0)
+		if err != nil {
+			return nil, err
+		}
+		t.data = trie.NewSecure(tr, t.db)
+	}
+	return t.data, nil
+}
+
 // Get returns the value for key stored in the trie.
 // The value bytes must not be modified by the caller.
 func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error) {
 	err = t.do(ctx, key, func() (err error) {
-		if t.trie == nil {
+		var st *trie.SecureTrie
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
+		st, err = t.getMap()
-		}
 		if err == nil {
-			res, err = t.trie.TryGet(key)
+			res, err = st.TryGet(key)
 		}
 		return
 	})
@@ -97,11 +108,10 @@ func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error)
 // stored in the trie.
 func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
 	err = t.do(ctx, key, func() (err error) {
-		if t.trie == nil {
+		var st *trie.SecureTrie
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
+		st, err = t.getMap()
-		}
 		if err == nil {
-			err = t.trie.TryUpdate(key, value)
+			err = st.TryUpdate(key, value)
 		}
 		return
 	})
@@ -111,11 +121,10 @@ func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
 // Delete removes any existing value for key from the trie.
 func (t *LightTrie) Delete(ctx context.Context, key []byte) (err error) {
 	err = t.do(ctx, key, func() (err error) {
-		if t.trie == nil {
+		var st *trie.SecureTrie
-			t.trie, err = trie.NewSecure(t.originalRoot, t.db, 0)
+		st, err = t.getMap()
-		}
 		if err == nil {
-			err = t.trie.TryDelete(key)
+			err = st.TryDelete(key)
 		}
 		return
 	})

metrics/metrics.go

@@ -75,6 +75,15 @@ func NewTimer(name string) metrics.Timer {
 	return metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry)
 }
 
+// NewCounter create a new metrics Counter, either a real one of a NOP stub depending
+// on the metrics flag.
+func NewCounter(name string) metrics.Counter {
+	if !Enabled {
+		return new(metrics.NilCounter)
+	}
+	return metrics.GetOrRegisterCounter(name, metrics.DefaultRegistry)
+}
+
 // CollectProcessMetrics periodically collects various metrics about the running
 // process.
 func CollectProcessMetrics(refresh time.Duration) {

miner/worker.go

@@ -583,7 +583,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB
 			continue
 		}
 		// Start executing the transaction
-		env.state.StartRecord(tx.Hash(), common.Hash{}, env.tcount)
+		env.state.SetTxContext(tx.Hash(), env.tcount)
 
 		err, logs := env.commitTransaction(tx, bc, gp)
 		switch {

trie/directcache.go

@@ -0,0 +1,319 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"fmt"
+	"errors"
+	"time"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+var (
+	directCacheLock              = &sync.Mutex{}
+	directCacheLocked            = false
+	directCacheWrites            = metrics.NewCounter("directcache/writes")
+	directCacheHits              = metrics.NewCounter("directcache/hits")
+	directCacheMisses            = metrics.NewCounter("directcache/misses")
+	directCacheTimer             = metrics.NewTimer("directcache/timer")
+	NotFound                     = errors.New("Cache entry not found")
+    MigrationPrefix              = []byte("directstatecachemigration:")
+)
+
+type MigrationStatus int
+
+const (
+	NotStarted = iota
+	Running    = iota
+	Complete   = iota
+)
+
+// CacheValidator can check whether a certain block is in the current canonical chain.
+type CacheValidator interface {
+	IsCanonChainBlock(uint64, common.Hash) bool
+}
+
+type DirectCache struct {
+	data      PersistentMap
+	db        Database
+	keyPrefix []byte
+	blockNum  uint64
+	blockHash common.Hash
+	validator CacheValidator
+	complete  bool
+	dirty     map[string]bool
+}
+
+type NullCacheValidator struct{}
+
+func (cv *NullCacheValidator) IsCanonChainBlock(num uint64, hash common.Hash) bool {
+	return false
+}
+
+func NewDirectCache(pm PersistentMap, db Database, keyPrefix []byte, blockNum uint64, blockHash common.Hash, validator CacheValidator, complete bool) *DirectCache {
+	return &DirectCache{
+		data:      pm,
+		db:        db,
+		keyPrefix: keyPrefix,
+		blockNum:  blockNum,
+		blockHash: blockHash,
+		validator: validator,
+		complete:  complete,
+		dirty:     make(map[string]bool),
+	}
+}
+
+func (dc *DirectCache) Iterator() *Iterator {
+	// Todo: If complete is true, implement an iterator over the DB instead.
+	return dc.data.Iterator()
+}
+
+func (dc *DirectCache) Get(key []byte) []byte {
+	res, err := dc.TryGet(key)
+	if err != nil && glog.V(logger.Error) {
+		glog.Errorf("Unhandled error: %v", err)
+	}
+	return res
+}
+
+func (dc *DirectCache) TryGet(key []byte) ([]byte, error) {
+	dirty := dc.dirty[string(key)]
+
+	// Use the underlying object for dirty keys
+	if !dirty {
+		if cached, ok := dc.getCached(key); ok {
+			return cached, nil
+		}
+	}
+
+	value, err := dc.data.TryGet(key)
+	if err != nil {
+		return value, err
+	}
+
+	if !dc.dirty[string(key)] {
+		// Flag the key as dirty so it gets written at commit time
+		dc.dirty[string(key)] = true
+	}
+
+	// Don't count fetches of dirty data as cache misses
+	if !dirty {
+		directCacheMisses.Inc(1)
+	}
+
+	return value, nil
+}
+
+func (dc *DirectCache) getCached(key []byte) ([]byte, bool) {
+	data, blockNum, blockHash, err := GetDirectCache(dc.keyPrefix, key, dc.db)
+	if err != nil {
+		if err == NotFound {
+			return nil, dc.complete
+		}
+		glog.Errorf("Error retrieving direct cache data: %v", err)
+		return nil, false
+	}
+
+	canonical := dc.blockNum > 0 && blockNum < dc.blockNum && dc.validator.IsCanonChainBlock(blockNum, blockHash)
+	return data, canonical
+}
+
+func (dc *DirectCache) Update(key, value []byte) {
+	if err := dc.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
+		glog.Errorf("Unhandled error: %v", err)
+	}
+}
+
+func (dc *DirectCache) TryUpdate(key, value []byte) error {
+	dc.dirty[string(key)] = true
+	return dc.data.TryUpdate(key, value)
+}
+
+func (dc *DirectCache) Delete(key []byte) {
+	if err := dc.TryDelete(key); err != nil && glog.V(logger.Error) {
+		glog.Errorf("Unhandled error: %v", err)
+	}
+}
+
+func (dc *DirectCache) TryDelete(key []byte) error {
+	dc.dirty[string(key)] = true
+	return dc.data.TryDelete(key)
+}
+
+func (dc *DirectCache) CommitTo(dbw DatabaseWriter) (root common.Hash, err error) {
+	if err := DirectCacheTransaction(func() error {
+		for k, _ := range dc.dirty {
+			v, err := dc.data.TryGet([]byte(k))
+			if _, ok := err.(*MissingNodeError); err != nil && !ok {
+				return err
+			}
+			if err := WriteDirectCache(dc.keyPrefix, []byte(k), v, dc.blockNum, dc.blockHash, dbw); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return common.Hash{}, err
+	}
+
+	dc.dirty = make(map[string]bool)
+	return dc.data.CommitTo(dbw)
+}
+
+// Populate iterates over the underlying trie, filling in any unset cache entries.
+// After Populate has completed, future DirectCache instances can have `complete`
+// set to true, for better efficiency on cache misses.
+func (dc *DirectCache) Populate() (err error) {
+	i := 0
+	writes := 0
+	it := dc.Iterator()
+	for it.Next() {
+		if err := DirectCacheTransaction(func() error {
+			if !HasDirectCache(dc.keyPrefix, it.Key, dc.db) {
+				writes += 1
+				if err = WriteDirectCache(dc.keyPrefix, it.Key, it.Value, dc.blockNum, dc.blockHash, dc.db); err != nil {
+					return err
+				}
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		i += 1
+		if i % 10000 == 0 && glog.V(logger.Info) {
+			glog.V(logger.Info).Infof("Constructing direct cache: processed %v entries, written %v", i, writes)
+		}
+	}
+	return nil
+}
+
+type cachedValue struct {
+	Value     []byte
+	BlockNum  uint64
+	BlockHash common.Hash
+}
+
+func DirectCacheTransaction(tx func() (error)) error {
+	directCacheLock.Lock()
+	directCacheLocked = true
+	defer func() {
+		directCacheLocked = false
+		directCacheLock.Unlock()
+	}()
+
+	return tx()
+}
+
+// WriteDirectCache places a value node directly into the database along with
+// block metadata to validate its relevancy.
+//
+// The method is meant to be used by code that circumvents the state database
+// and its integrated cache, namely during fast sync and database upgrades.
+func WriteDirectCache(prefix, key, value []byte, number uint64, hash common.Hash, dbw DatabaseWriter) error {
+	directCacheWrites.Inc(1)
+
+	if !directCacheLocked {
+		return fmt.Errorf("WriteDirectCache may only be called in a DirectCacheTransaction")
+	}
+
+	enc, _ := rlp.EncodeToBytes(cachedValue{value, number, hash})
+	return dbw.Put(append(prefix, key...), enc)
+}
+
+// GetDirectCache retrieves a value node directly from the database along with
+// block metadata to validate its relevancy.
+//
+// The method is meant to be used by code that circumvents the state database
+// and its integrated cache, namely during fast sync and database upgrades.
+func GetDirectCache(prefix, key []byte, db Database) ([]byte, uint64, common.Hash, error) {
+	defer func(start time.Time) { directCacheTimer.UpdateSince(start) }(time.Now())
+
+	enc, _ := db.Get(append(prefix, key...))
+	if len(enc) == 0 {
+		return nil, 0, common.Hash{}, NotFound
+	}
+
+	var data cachedValue
+	if err := rlp.DecodeBytes(enc, &data); err != nil {
+		return nil, 0, common.Hash{}, fmt.Errorf("Can't decode cached object at %x: %v", key, err)
+	}
+	return data.Value, data.BlockNum, data.BlockHash, nil
+}
+
+// HasDirectCache returns true iff a direct cache node exists for the specified key
+func HasDirectCache(prefix, key []byte, db Database) bool {
+	if enc, err := db.Get(append(prefix, key...)); err == nil && len(enc) > 0 {
+		return true
+	}
+	return false
+}
+
+type migrationState struct {
+	Status MigrationStatus
+	Hash   common.Hash
+}
+
+// GetMigrationState returns the block number of the migration to the direct cache, and
+// whether or not it's complete.
+func GetMigrationState(prefix []byte, db Database) (common.Hash, MigrationStatus) {
+	enc, _ := db.Get(append(MigrationPrefix, prefix...))
+	if len(enc) == 0 {
+		return common.Hash{}, NotStarted
+	}
+
+	var data migrationState
+	if err := rlp.DecodeBytes(enc, &data); err != nil {
+		glog.Errorf("Could not decode migration status: %v", err)
+		return common.Hash{}, NotStarted
+	}
+
+	return data.Hash, data.Status
+}
+
+// SetMigrationState updates the migration state in the database
+func SetMigrationState(prefix []byte, blockHash common.Hash, status MigrationStatus, db Database) error {
+	enc, _ := rlp.EncodeToBytes(migrationState{status, blockHash})
+	return db.Put(append(MigrationPrefix, prefix...), enc)
+}
+
+// DirectCacheReads retrieves a global counter measuring the number of direct
+// cache reads from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheReads() int64 {
+	return directCacheTimer.Count()
+}
+
+// DirectCacheWrites retrieves a global counter measuring the number of direct
+// cache writes from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheWrites() int64 {
+	return directCacheWrites.Count()
+}
+
+// DirectCacheMisses retrieves a global counter measuring the number of direct
+// cache misses from the disk since process startup. This isn't useful for anything
+// apart from trie debugging purposes.
+func DirectCacheMisses() int64 {
+	return directCacheMisses.Count()
+}

trie/iterator.go

@@ -77,6 +77,7 @@ type nodeIteratorState struct {
 	hash    common.Hash // Hash of the node being iterated (nil if not standalone)
 	node    node        // Trie node being iterated
 	parent  common.Hash // Hash of the first full ancestor node (nil if current is the root)
+	nibbles []byte      // Key nibbles leading up to this node for key reconstruction
 	child   int         // Child to be processed next
 }
 
@@ -88,6 +89,7 @@ type NodeIterator struct {
 	Hash     common.Hash // Hash of the current node being iterated (nil if not standalone)
 	Node     node        // Current node being iterated (internal representation)
 	Parent   common.Hash // Hash of the first full ancestor node (nil if current is the root)
+	Nibbles  []byte      // Key nibbles leading up to this node for key reconstruction
 	Leaf     bool        // Flag whether the current node is a value (data) node
 	LeafBlob []byte      // Data blob contained within a leaf (otherwise nil)
 
@@ -155,10 +157,15 @@ func (it *NodeIterator) step() error {
 			}
 			for parent.child++; parent.child < len(node.Children); parent.child++ {
 				if current := node.Children[parent.child]; current != nil {
+					nibbles := parent.nibbles
+					if parent.child < 16 {
+						nibbles = append(nibbles, byte(parent.child))
+					}
 					it.stack = append(it.stack, &nodeIteratorState{
 						hash:    common.BytesToHash(node.flags.hash),
 						node:    current,
 						parent:  ancestor,
+						nibbles: nibbles,
 						child:   -1,
 					})
 					break
@@ -174,6 +181,7 @@ func (it *NodeIterator) step() error {
 				hash:    common.BytesToHash(node.flags.hash),
 				node:    node.Val,
 				parent:  ancestor,
+				nibbles: append(parent.nibbles, node.Key...),
 				child:   -1,
 			})
 		} else if hash, ok := parent.node.(hashNode); ok {
@@ -191,6 +199,7 @@ func (it *NodeIterator) step() error {
 				hash:    common.BytesToHash(hash),
 				node:    node,
 				parent:  ancestor,
+				nibbles: parent.nibbles,
 				child:   -1,
 			})
 		} else {
@@ -207,7 +216,7 @@ func (it *NodeIterator) step() error {
 // The method returns whether there are any more data left for inspection.
 func (it *NodeIterator) retrieve() bool {
 	// Clear out any previously set values
-	it.Hash, it.Node, it.Parent, it.Leaf, it.LeafBlob = common.Hash{}, nil, common.Hash{}, false, nil
+	it.Hash, it.Node, it.Parent, it.Nibbles, it.Leaf, it.LeafBlob = common.Hash{}, nil, common.Hash{}, nil, false, nil
 
 	// If the iteration's done, return no available data
 	if it.trie == nil {
@@ -216,7 +225,7 @@ func (it *NodeIterator) retrieve() bool {
 	// Otherwise retrieve the current node and resolve leaf accessors
 	state := it.stack[len(it.stack)-1]
 
-	it.Hash, it.Node, it.Parent = state.hash, state.node, state.parent
+	it.Hash, it.Node, it.Parent, it.Nibbles = state.hash, state.node, state.parent, state.nibbles
 	if value, ok := it.Node.(valueNode); ok {
 		it.Leaf, it.LeafBlob = true, []byte(value)
 	}

trie/proof_test.go

@@ -50,7 +50,7 @@ func TestProof(t *testing.T) {
 }
 
 func TestOneElementProof(t *testing.T) {
-	trie := new(Trie)
+	trie, _ := New(common.Hash{}, nil, 0)
 	updateString(trie, "k", "v")
 	proof := trie.Prove([]byte("k"))
 	if proof == nil {
@@ -130,7 +130,7 @@ func BenchmarkVerifyProof(b *testing.B) {
 }
 
 func randomTrie(n int) (*Trie, map[string]*kv) {
-	trie := new(Trie)
+	trie, _ := New(common.Hash{}, nil, 0)
 	vals := make(map[string]*kv)
 	for i := byte(0); i < 100; i++ {
 		value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}

trie/secure_trie.go

@@ -26,6 +26,17 @@ var secureKeyPrefix = []byte("secure-key-")
 
 const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
 
+type PersistentMap interface {
+	Iterator() *Iterator
+	Get(key []byte) []byte
+	TryGet(key []byte) ([]byte, error)
+	Update(key, value []byte)
+	TryUpdate(key, value []byte) error
+	Delete(key []byte)
+	TryDelete(key []byte) error
+	CommitTo(db DatabaseWriter) (root common.Hash, err error)
+}
+
 // SecureTrie wraps a trie with key hashing. In a secure trie, all
 // access operations hash the key using keccak256. This prevents
 // calling code from creating long chains of nodes that
@@ -37,75 +48,65 @@ const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
 //
 // SecureTrie is not safe for concurrent use.
 type SecureTrie struct {
-	trie             Trie
+	data             PersistentMap
+	db               Database
 	hashKeyBuf       [secureKeyLength]byte
 	secKeyBuf        [200]byte
 	secKeyCache      map[string][]byte
 	secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
 }
 
-// NewSecure creates a trie with an existing root node from db.
+// NewSecure creates a secure persistent map from an existing map.
-//
+func NewSecure(pm PersistentMap, db Database) *SecureTrie {
-// If root is the zero hash or the sha3 hash of an empty string, the
+	if pm == nil {
-// trie is initially empty. Otherwise, New will panic if db is nil
+		panic("NewSecure called with nil persistent map")
-// and returns MissingNodeError if the root node cannot be found.
+	}
-//
-// Accessing the trie loads nodes from db on demand.
-// Loaded nodes are kept around until their 'cache generation' expires.
-// A new cache generation is created by each call to Commit.
-// cachelimit sets the number of past cache generations to keep.
-func NewSecure(root common.Hash, db Database, cachelimit uint16) (*SecureTrie, error) {
 	if db == nil {
 		panic("NewSecure called with nil database")
 	}
-	trie, err := New(root, db)
+	return &SecureTrie{data: pm, db: db}
-	if err != nil {
-		return nil, err
-	}
-	trie.SetCacheLimit(cachelimit)
-	return &SecureTrie{trie: *trie}, nil
 }
 
-// Get returns the value for key stored in the trie.
+// Get returns the value for key stored in the map.
 // The value bytes must not be modified by the caller.
 func (t *SecureTrie) Get(key []byte) []byte {
 	res, err := t.TryGet(key)
 	if err != nil && glog.V(logger.Error) {
-		glog.Errorf("Unhandled trie error: %v", err)
+		glog.Errorf("Unhandled persistent map error: %v", err)
 	}
 	return res
 }
 
-// TryGet returns the value for key stored in the trie.
+// TryGet returns the value for key stored in the map.
 // The value bytes must not be modified by the caller.
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
-	return t.trie.TryGet(t.hashKey(key))
+	return t.data.TryGet(t.hashKey(key))
 }
 
-// Update associates key with value in the trie. Subsequent calls to
+// Update associates key with value in the map. Subsequent calls to
 // Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// is deleted from the map and calls to Get will return nil.
 //
 // The value bytes must not be modified by the caller while they are
-// stored in the trie.
+// stored in the map.
 func (t *SecureTrie) Update(key, value []byte) {
 	if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
-		glog.Errorf("Unhandled trie error: %v", err)
+		glog.Errorf("Unhandled persistent map error: %v", err)
 	}
 }
 
-// TryUpdate associates key with value in the trie. Subsequent calls to
+// TryUpdate associates key with value in the map. Subsequent calls to
 // Get will return value. If value has length zero, any existing value
-// is deleted from the trie and calls to Get will return nil.
+// is deleted from the map and calls to Get will return nil.
 //
 // The value bytes must not be modified by the caller while they are
-// stored in the trie.
+// stored in the map.
 //
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryUpdate(key, value []byte) error {
 	hk := t.hashKey(key)
-	err := t.trie.TryUpdate(hk, value)
+	err := t.data.TryUpdate(hk, value)
 	if err != nil {
 		return err
 	}
@@ -113,19 +114,19 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error {
 	return nil
 }
 
-// Delete removes any existing value for key from the trie.
+// Delete removes any existing value for key from the map.
 func (t *SecureTrie) Delete(key []byte) {
 	if err := t.TryDelete(key); err != nil && glog.V(logger.Error) {
-		glog.Errorf("Unhandled trie error: %v", err)
+		glog.Errorf("Unhandled persistent map error: %v", err)
 	}
 }
 
-// TryDelete removes any existing value for key from the trie.
+// TryDelete removes any existing value for key from the map.
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryDelete(key []byte) error {
 	hk := t.hashKey(key)
 	delete(t.getSecKeyCache(), string(hk))
-	return t.trie.TryDelete(hk)
+	return t.data.TryDelete(hk)
 }
 
 // GetKey returns the sha3 preimage of a hashed key that was
@@ -134,51 +135,37 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
 	if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
 		return key
 	}
-	key, _ := t.trie.db.Get(t.secKey(shaKey))
+	key, _ := t.db.Get(t.secKey(shaKey))
 	return key
 }
 
-// Commit writes all nodes and the secure hash pre-images to the trie's database.
-// Nodes are stored with their sha3 hash as the key.
-//
-// Committing flushes nodes from memory. Subsequent Get calls will load nodes
-// from the database.
-func (t *SecureTrie) Commit() (root common.Hash, err error) {
-	return t.CommitTo(t.trie.db)
-}
-
-func (t *SecureTrie) Hash() common.Hash {
-	return t.trie.Hash()
-}
-
-func (t *SecureTrie) Root() []byte {
-	return t.trie.Root()
-}
-
 func (t *SecureTrie) Iterator() *Iterator {
-	return t.trie.Iterator()
+	return t.data.Iterator()
-}
-
-func (t *SecureTrie) NodeIterator() *NodeIterator {
-	return NewNodeIterator(&t.trie)
 }
 
 // CommitTo writes all nodes and the secure hash pre-images to the given database.
 // Nodes are stored with their sha3 hash as the key.
 //
 // Committing flushes nodes from memory. Subsequent Get calls will load nodes from
-// the trie's database. Calling code must ensure that the changes made to db are
+// the database. Calling code must ensure that the changes made to db are
-// written back to the trie's attached database before using the trie.
+// written back to the attached database before using the map.
 func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
+	if err := t.CommitPreimages(); err != nil {
+		return common.Hash{}, err
+	}
+	return t.data.CommitTo(db)
+}
+
+func (t *SecureTrie) CommitPreimages() error {
 	if len(t.getSecKeyCache()) > 0 {
 		for hk, key := range t.secKeyCache {
-			if err := db.Put(t.secKey([]byte(hk)), key); err != nil {
+			if err := t.db.Put(t.secKey([]byte(hk)), key); err != nil {
-				return common.Hash{}, err
+				return err
 			}
 		}
 		t.secKeyCache = make(map[string][]byte)
 	}
-	return t.trie.CommitTo(db)
+	return nil
 }
 
 // secKey returns the database key for the preimage of key, as an ephemeral buffer.
@@ -203,7 +190,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte {
 }
 
 // getSecKeyCache returns the current secure key cache, creating a new one if
-// ownership changed (i.e. the current secure trie is a copy of another owning
+// ownership changed (i.e. the current secure map is a copy of another owning
 // the actual cache).
 func (t *SecureTrie) getSecKeyCache() map[string][]byte {
 	if t != t.secKeyCacheOwner {

trie/secure_trie_test.go

@@ -27,17 +27,19 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-func newEmptySecure() *SecureTrie {
+func newEmptySecure() (*Trie, *SecureTrie) {
 	db, _ := ethdb.NewMemDatabase()
-	trie, _ := NewSecure(common.Hash{}, db, 0)
+	tr, _ := New(common.Hash{}, db, 0)
-	return trie
+	st := NewSecure(tr, db)
+	return tr, st
 }
 
 // makeTestSecureTrie creates a large enough secure trie for testing.
 func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
 	// Create an empty trie
 	db, _ := ethdb.NewMemDatabase()
-	trie, _ := NewSecure(common.Hash{}, db, 0)
+	tr, _ := New(common.Hash{}, db, 0)
+	trie := NewSecure(tr, db)
 
 	// Fill it with some arbitrary data
 	content := make(map[string][]byte)
@@ -58,14 +60,14 @@ func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
 			trie.Update(key, val)
 		}
 	}
-	trie.Commit()
+	trie.CommitTo(db)
 
 	// Return the generated trie
 	return db, trie, content
 }
 
 func TestSecureDelete(t *testing.T) {
-	trie := newEmptySecure()
+	trie, st := newEmptySecure()
 	vals := []struct{ k, v string }{
 		{"do", "verb"},
 		{"ether", "wookiedoo"},
@@ -78,9 +80,9 @@ func TestSecureDelete(t *testing.T) {
 	}
 	for _, val := range vals {
 		if val.v != "" {
-			trie.Update([]byte(val.k), []byte(val.v))
+			st.Update([]byte(val.k), []byte(val.v))
 		} else {
-			trie.Delete([]byte(val.k))
+			st.Delete([]byte(val.k))
 		}
 	}
 	hash := trie.Hash()
@@ -91,24 +93,24 @@ func TestSecureDelete(t *testing.T) {
 }
 
 func TestSecureGetKey(t *testing.T) {
-	trie := newEmptySecure()
+	_, st := newEmptySecure()
-	trie.Update([]byte("foo"), []byte("bar"))
+	st.Update([]byte("foo"), []byte("bar"))
 
 	key := []byte("foo")
 	value := []byte("bar")
 	seckey := crypto.Keccak256(key)
 
-	if !bytes.Equal(trie.Get(key), value) {
+	if !bytes.Equal(st.Get(key), value) {
 		t.Errorf("Get did not return bar")
 	}
-	if k := trie.GetKey(seckey); !bytes.Equal(k, key) {
+	if k := st.GetKey(seckey); !bytes.Equal(k, key) {
 		t.Errorf("GetKey returned %q, want %q", k, key)
 	}
 }
 
 func TestSecureTrieConcurrency(t *testing.T) {
 	// Create an initial trie and copy if for concurrent access
-	_, trie, _ := makeTestSecureTrie()
+	db, trie, _ := makeTestSecureTrie()
 
 	threads := runtime.NumCPU()
 	tries := make([]*SecureTrie, threads)
@@ -137,7 +139,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
 					tries[index].Update(key, val)
 				}
 			}
-			tries[index].Commit()
+			tries[index].CommitTo(db)
 		}(i)
 	}
 	// Wait for all threads to finish

trie/sync.go

@@ -36,10 +36,39 @@ type request struct {
 	raw  bool        // Whether this is a raw entry (code) or a trie node
 
 	parents []*request // Parent state nodes referencing this entry (notify all upon completion)
+	nibbles [][]byte   // Trie path nibbles that accumulate up to the key at the leaf (paired with parents)
 	depth   int        // Depth level within the trie the node is located to prioritise DFS
 	deps    int        // Number of dependencies before allowed to commit this node
 
 	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
+	dcPrefix []byte               // Database prefix to use for leaf direct caching
+}
+
+// keys traverses the request ancestry tree, reconstructing the key paths leading
+// to the current request through the nibbles represented by each node.
+func (r *request) keys(suffix []byte) [][]byte {
+	keys := r.paths(suffix)
+	for i := 0; i < len(keys); i++ {
+		keys[i] = compactHexEncode(keys[i])
+	}
+	return keys
+}
+
+// paths traverses the request ancestry tree, reconstructing the key paths leading
+// to the current request through the nibbles represented by each node.
+func (r *request) paths(suffix []byte) [][]byte {
+	// A root level request just returns the trailing suffix
+	if len(r.parents) == 0 {
+		return [][]byte{suffix}
+	}
+	// Otherwise collect all the ancestor paths, and append the current one
+	var keys [][]byte
+	for i, parent := range r.parents {
+		for _, key := range parent.paths(r.nibbles[i]) {
+			keys = append(keys, append(key, suffix...))
+		}
+	}
+	return keys
 }
 
 // SyncResult is a simple list to return missing nodes along with their request
@@ -52,7 +81,7 @@ type SyncResult struct {
 // TrieSyncLeafCallback is a callback type invoked when a trie sync reaches a
 // leaf node. It's used by state syncing to check if the leaf node requires some
 // further data syncing.
-type TrieSyncLeafCallback func(leaf []byte, parent common.Hash) error
+type TrieSyncLeafCallback func(keys [][]byte, leaf []byte, parent common.Hash) error
 
 // TrieSync is the main state trie synchronisation scheduler, which provides yet
 // unknown trie hashes to retrieve, accepts node data associated with said hashes
@@ -64,18 +93,18 @@ type TrieSync struct {
 }
 
 // NewTrieSync creates a new trie data download scheduler.
-func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
+func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback, dcPrefix []byte) *TrieSync {
 	ts := &TrieSync{
 		database: database,
 		requests: make(map[common.Hash]*request),
 		queue:    prque.New(),
 	}
-	ts.AddSubTrie(root, 0, common.Hash{}, callback)
+	ts.AddSubTrie(root, 0, common.Hash{}, callback, dcPrefix)
 	return ts
 }
 
 // AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
-func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
+func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback, dcPrefix []byte) {
 	// Short circuit if the trie is empty or already known
 	if root == emptyRoot {
 		return
@@ -90,6 +119,7 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
 		hash:     root,
 		depth:    depth,
 		callback: callback,
+		dcPrefix: dcPrefix,
 	}
 	// If this sub-trie has a designated parent, link them together
 	if parent != (common.Hash{}) {
@@ -198,6 +228,7 @@ func (s *TrieSync) schedule(req *request) {
 	// If we're already requesting this node, add a new reference and stop
 	if old, ok := s.requests[req.hash]; ok {
 		old.parents = append(old.parents, req.parents...)
+		old.nibbles = append(old.nibbles, req.nibbles...)
 		return
 	}
 	// Schedule the request for future retrieval
@@ -210,6 +241,7 @@ func (s *TrieSync) schedule(req *request) {
 func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 	// Gather all the children of the node, irrelevant whether known or not
 	type child struct {
+		path  []byte
 		node  node
 		depth int
 	}
@@ -218,13 +250,19 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 	switch node := (object).(type) {
 	case *shortNode:
 		children = []child{{
+			path:  node.Key,
 			node:  node.Val,
 			depth: req.depth + len(node.Key),
 		}}
 	case *fullNode:
 		for i := 0; i < 17; i++ {
 			if node.Children[i] != nil {
+				var path []byte
+				if i < 16 {
+					path = []byte{byte(i)}
+				}
 				children = append(children, child{
+					path:  path,
 					node:  node.Children[i],
 					depth: req.depth + 1,
 				})
@@ -239,7 +277,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 		// Notify any external watcher of a new key/value node
 		if req.callback != nil {
 			if node, ok := (child.node).(valueNode); ok {
-				if err := req.callback(node, req.hash); err != nil {
+				if err := req.callback(req.keys(child.path), node, req.hash); err != nil {
 					return nil, err
 				}
 			}
@@ -249,14 +287,40 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
 			// Try to resolve the node from the local database
 			blob, _ := s.database.Get(node)
 			if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil {
+				// Local node found, iterate and invoke the leaf callback for all subleaves
+				if req.callback != nil && req.dcPrefix != nil {
+					trie, _ := New(common.BytesToHash(node[:]), s.database, 0)
+					unknown := false
+
+					it := NewNodeIterator(trie)
+					for it.Next() {
+						if it.Leaf {
+							// If this branch was already visited, abort iteration
+							keys := req.keys(append(child.path, it.Nibbles...))
+							if !unknown {
+								if HasDirectCache(req.dcPrefix, keys[0], s.database) {
+									break
+								}
+								// Branch is unknown, mark as so to avoid repeated db lookups
+								unknown = true
+							}
+							// Otherwise write out all accounts ending in this leaf
+							if err := req.callback(keys, nil, req.hash); err != nil {
+								return nil, err
+							}
+						}
+					}
+				}
 				continue
 			}
 			// Locally unknown node, schedule for retrieval
 			requests = append(requests, &request{
 				hash:     common.BytesToHash(node),
 				parents:  []*request{req},
+				nibbles:  [][]byte{child.path},
 				depth:    child.depth,
 				callback: req.callback,
+				dcPrefix: req.dcPrefix,
 			})
 		}
 	}

trie/sync_test.go

@@ -28,7 +28,7 @@ import (
 func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
 	// Create an empty trie
 	db, _ := ethdb.NewMemDatabase()
-	trie, _ := New(common.Hash{}, db)
+	trie, _ := New(common.Hash{}, db, 0)
 
 	// Fill it with some arbitrary data
 	content := make(map[string][]byte)
@@ -55,11 +55,33 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
 	return db, trie, content
 }
 
+// makeRepetitiveTestTrie creates a test trie to test node-wise reconstruction,
+// where all values are the same, forcing large parts of the trie to share nodes.
+func makeRepetitiveTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
+	// Create an empty trie
+	db, _ := ethdb.NewMemDatabase()
+	trie, _ := New(common.Hash{}, db, 0)
+
+	// Fill it with some arbitrary data
+	content := make(map[string][]byte)
+	for i := byte(0); i < 255; i++ {
+		for j := byte(0); j < 255; j++ {
+			key, val := common.LeftPadBytes([]byte{j, i}, 32), common.LeftPadBytes(nil, 32)
+			content[string(key)] = val
+			trie.Update(key, val)
+		}
+	}
+	trie.Commit()
+
+	// Return the generated trie
+	return db, trie, content
+}
+
 // checkTrieContents cross references a reconstructed trie with an expected data
 // content map.
 func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
 	// Check root availability and trie contents
-	trie, err := New(common.BytesToHash(root), db)
+	trie, err := New(common.BytesToHash(root), db, 0)
 	if err != nil {
 		t.Fatalf("failed to create trie at %x: %v", root, err)
 	}
@@ -76,7 +98,7 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
 // checkTrieConsistency checks that all nodes in a trie are indeed present.
 func checkTrieConsistency(db Database, root common.Hash) error {
 	// Create and iterate a trie rooted in a subnode
-	trie, err := New(root, db)
+	trie, err := New(root, db, 0)
 	if err != nil {
 		return nil // // Consider a non existent state consistent
 	}
@@ -88,12 +110,12 @@ func checkTrieConsistency(db Database, root common.Hash) error {
 
 // Tests that an empty trie is not scheduled for syncing.
 func TestEmptyTrieSync(t *testing.T) {
-	emptyA, _ := New(common.Hash{}, nil)
+	emptyA, _ := New(common.Hash{}, nil, 0)
-	emptyB, _ := New(emptyRoot, nil)
+	emptyB, _ := New(emptyRoot, nil, 0)
 
 	for i, trie := range []*Trie{emptyA, emptyB} {
 		db, _ := ethdb.NewMemDatabase()
-		if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
+		if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil, nil).Missing(1); len(req) != 0 {
 			t.Errorf("test %d: content requested for empty trie: %v", i, req)
 		}
 	}
@@ -110,7 +132,7 @@ func testIterativeTrieSync(t *testing.T, batch int) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(batch)...)
 	for len(queue) > 0 {
@@ -139,7 +161,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(10000)...)
 	for len(queue) > 0 {
@@ -173,7 +195,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(batch) {
@@ -210,7 +232,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(10000) {
@@ -253,7 +275,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(0)...)
 	requested := make(map[common.Hash]struct{})
@@ -289,7 +311,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 
 	// Create a destination trie and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil, nil)
 
 	added := []common.Hash{}
 	queue := append([]common.Hash{}, sched.Missing(1)...)
@@ -331,3 +353,49 @@ func TestIncompleteTrieSync(t *testing.T) {
 		dstDb.Put(key, value)
 	}
 }
+
+// Tests that if a leaf callback is specified, all leaf nodes are invoked with
+// it, even if sync short circuits with existing local nodes.
+func TestAllLeafCallbackTrieSync(t *testing.T) {
+	// Create a random trie to copy
+	srcDb, srcTrie, srcData := makeRepetitiveTestTrie()
+
+	// Create a callback to accumulate all keys for later verification
+	leaves, count := make(map[string]struct{}), 0
+	callback := func(keys [][]byte, leaf []byte, parent common.Hash) error {
+		for _, key := range keys {
+			leaves[string(key)] = struct{}{}
+			count++
+		}
+		return nil
+	}
+	// Create a destination trie and sync with the scheduler
+	dstDb, _ := ethdb.NewMemDatabase()
+	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, callback, []byte("leaf:"))
+
+	queue := append([]common.Hash{}, sched.Missing(1)...)
+	for len(queue) > 0 {
+		results := make([]SyncResult, len(queue))
+		for i, hash := range queue {
+			data, err := srcDb.Get(hash.Bytes())
+			if err != nil {
+				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+			}
+			results[i] = SyncResult{hash, data}
+		}
+		if index, err := sched.Process(results); err != nil {
+			t.Fatalf("failed to process result #%d: %v", index, err)
+		}
+		queue = append(queue[:0], sched.Missing(1)...)
+	}
+	// Cross check that the two tries are in sync
+	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+
+	// Ensure the leaf callback was invoked for all leaves, including cached ones
+	if len(leaves) != len(srcData) {
+		t.Errorf("Leaf callback count mismatch: have %v, want %v", len(leaves), len(srcData))
+	}
+	if len(leaves) != count {
+		t.Errorf("Duplicate leaf callbacks: have %v, want %v", count, len(leaves))
+	}
+}

trie/trie.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// Package trie implements Merkle Patricia Tries.
+// Package trie implements Merkle Patricia tries.
 package trie
 
 import (
@@ -73,11 +73,11 @@ type DatabaseWriter interface {
 	Put(key, value []byte) error
 }
 
-// Trie is a Merkle Patricia Trie.
+// trie is a Merkle Patricia trie.
 // The zero value is an empty trie with no database.
 // Use New to create a trie that sits on top of a database.
 //
-// Trie is not safe for concurrent use.
+// trie is not safe for concurrent use.
 type Trie struct {
 	root         node
 	db           Database
@@ -90,12 +90,6 @@ type Trie struct {
 	cachegen, cachelimit uint16
 }
 
-// SetCacheLimit sets the number of 'cache generations' to keep.
-// A cache generations is created by a call to Commit.
-func (t *Trie) SetCacheLimit(l uint16) {
-	t.cachelimit = l
-}
-
 // newFlag returns the cache flag value for a newly created node.
 func (t *Trie) newFlag() nodeFlag {
 	return nodeFlag{dirty: true, gen: t.cachegen}
@@ -107,11 +101,14 @@ func (t *Trie) newFlag() nodeFlag {
 // trie is initially empty and does not require a database. Otherwise,
 // New will panic if db is nil and returns a MissingNodeError if root does
 // not exist in the database. Accessing the trie loads nodes from db on demand.
-func New(root common.Hash, db Database) (*Trie, error) {
+//
-	trie := &Trie{db: db, originalRoot: root}
+// cacheLimit is the number of 'cache generations' to keep.
+// A cache generations is created by a call to Commit.
+func New(root common.Hash, db Database, cacheLimit uint16) (*Trie, error) {
+	trie := &Trie{db: db, originalRoot: root, cachelimit: cacheLimit}
 	if (root != common.Hash{}) && root != emptyRoot {
 		if db == nil {
-			panic("trie.New: cannot use existing root without a database")
+			panic("Trie.New: cannot use existing root without a database")
 		}
 		rootnode, err := trie.resolveHash(root[:], nil, nil)
 		if err != nil {

trie/trie_test.go

@@ -40,7 +40,7 @@ func init() {
 // Used for testing
 func newEmpty() *Trie {
 	db, _ := ethdb.NewMemDatabase()
-	trie, _ := New(common.Hash{}, db)
+	trie, _ := New(common.Hash{}, db, 0)
 	return trie
 }
 
@@ -63,7 +63,7 @@ func TestNull(t *testing.T) {
 
 func TestMissingRoot(t *testing.T) {
 	db, _ := ethdb.NewMemDatabase()
-	trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db)
+	trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db, 0)
 	if trie != nil {
 		t.Error("New returned non-nil trie for invalid root")
 	}
@@ -74,36 +74,36 @@ func TestMissingRoot(t *testing.T) {
 
 func TestMissingNode(t *testing.T) {
 	db, _ := ethdb.NewMemDatabase()
-	trie, _ := New(common.Hash{}, db)
+	trie, _ := New(common.Hash{}, db, 0)
 	updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
 	updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
 	root, _ := trie.Commit()
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err := trie.TryGet([]byte("120000"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err = trie.TryGet([]byte("120099"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err = trie.TryGet([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	err = trie.TryDelete([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
@@ -111,31 +111,31 @@ func TestMissingNode(t *testing.T) {
 
 	db.Delete(common.FromHex("e1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9"))
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err = trie.TryGet([]byte("120000"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err = trie.TryGet([]byte("120099"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	_, err = trie.TryGet([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
 
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 0)
 	err = trie.TryDelete([]byte("123456"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
@@ -263,7 +263,7 @@ func TestReplication(t *testing.T) {
 	}
 
 	// create a new trie on top of the database and check that lookups work.
-	trie2, err := New(exp, trie.db)
+	trie2, err := New(exp, trie.db, 0)
 	if err != nil {
 		t.Fatalf("can't recreate trie at %x: %v", exp, err)
 	}
@@ -332,8 +332,7 @@ func TestCacheUnload(t *testing.T) {
 	// The branch containing it is loaded from DB exactly two times:
 	// in the 0th and 6th iteration.
 	db := &countingDB{Database: trie.db, gets: make(map[string]int)}
-	trie, _ = New(root, db)
+	trie, _ = New(root, db, 5)
-	trie.SetCacheLimit(5)
 	for i := 0; i < 12; i++ {
 		getString(trie, key1)
 		trie.Commit()
@@ -417,7 +416,7 @@ func randRead(r *rand.Rand, b []byte) {
 
 func runRandTest(rt randTest) bool {
 	db, _ := ethdb.NewMemDatabase()
-	tr, _ := New(common.Hash{}, db)
+	tr, _ := New(common.Hash{}, db, 0)
 	values := make(map[string]string) // tracks content of the trie
 
 	for _, step := range rt {
@@ -446,13 +445,13 @@ func runRandTest(rt randTest) bool {
 			if err != nil {
 				panic(err)
 			}
-			newtr, err := New(hash, db)
+			newtr, err := New(hash, db, 0)
 			if err != nil {
 				panic(err)
 			}
 			tr = newtr
 		case opItercheckhash:
-			checktr, _ := New(common.Hash{}, nil)
+			checktr, _ := New(common.Hash{}, nil, 0)
 			it := tr.Iterator()
 			for it.Next() {
 				checktr.Update(it.Key, it.Value)
@@ -520,10 +519,10 @@ func BenchmarkHashLE(b *testing.B)   { benchHash(b, binary.LittleEndian) }
 const benchElemCount = 20000
 
 func benchGet(b *testing.B, commit bool) {
-	trie := new(Trie)
+	trie, _ := New(common.Hash{}, nil, 0)
 	if commit {
 		_, tmpdb := tempDB()
-		trie, _ = New(common.Hash{}, tmpdb)
+		trie, _ = New(common.Hash{}, tmpdb, 0)
 	}
 	k := make([]byte, 32)
 	for i := 0; i < benchElemCount; i++ {