core, trie: finalize the new trie node-caching db layer

accounts/abi/bind/backends/simulated.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
@@ -103,7 +104,7 @@ func (b *SimulatedBackend) Rollback() {
 func (b *SimulatedBackend) rollback() {
 	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database, nil))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(trie.NewDatabase(b.database)))
 }
 
 // CodeAt returns the code associated with a certain account in the blockchain.
@@ -310,7 +311,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		block.AddTx(tx)
 	})
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database, nil))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(trie.NewDatabase(b.database)))
 	return nil
 }
 
@@ -387,7 +388,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database, nil))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(trie.NewDatabase(b.database)))
 
 	return nil
 }

cmd/evm/runner.go

@@ -37,6 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
@@ -96,11 +97,11 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
-		_, statedb = gen.ToBlock()
+		_, statedb, _ = gen.ToBlock()
 		chainConfig = gen.Config
 	} else {
 		db, _ := ethdb.NewMemDatabase()
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))

cmd/geth/chaincmd.go

@@ -379,7 +379,7 @@ func dump(ctx *cli.Context) error {
 			fmt.Println("{}")
 			utils.Fatalf("block not found")
 		} else {
-			state, err := state.New(block.Root(), state.NewDatabase(chainDb, nil))
+			state, err := state.New(block.Root(), state.NewDatabase(trie.NewDatabase(chainDb)))
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}

core/blockchain.go

@@ -102,6 +102,8 @@ type BlockChain struct {
 	blockCache   *lru.Cache     // Cache for the most recent entire blocks
 	futureBlocks *lru.Cache     // future blocks are blocks added for later processing
 
+	procTime time.Duration // Accumulator for measuring total block processing for the trie node dumping
+
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
 	// procInterrupt must be atomically called
@@ -129,7 +131,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
 	bc := &BlockChain{
 		config:       config,
 		chainDb:      chainDb,
-		stateCache:   state.NewDatabase(chainDb, trie.NewNodePool()),
+		stateCache:   state.NewDatabase(trie.NewDatabase(chainDb)),
 		quit:         make(chan struct{}),
 		bodyCache:    bodyCache,
 		bodyRLPCache: bodyRLPCache,
@@ -292,7 +294,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x…]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), bc.chainDb, nil, 0); err != nil {
+	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil {
 		return err
 	}
 	// If all checks out, manually set the head block
@@ -594,7 +596,7 @@ func (bc *BlockChain) Stop() {
 	root := bc.CurrentHeader().Root
 
 	batch := bc.chainDb.NewBatch()
-	if err := bc.stateCache.NodePool().Commit(root, batch); err != nil {
+	if err := bc.stateCache.TrieDB().Commit(root, batch); err != nil {
 		log.Error("Failed to commit latest state trie", "err", err)
 	}
 	if err := batch.Write(); err != nil {
@@ -776,6 +778,8 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	return 0, nil
 }
 
+var lastWrite uint64
+
 // WriteBlock writes the block to the chain.
 func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) {
 	bc.wg.Add(1)
@@ -802,18 +806,37 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
 	if err := WriteBlock(batch, block); err != nil {
 		return NonStatTy, err
 	}
-	root, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number()))
+	root, err := state.Commit(bc.config.IsEIP158(block.Number()))
 	if err != nil {
 		return NonStatTy, err
 	}
-	pool := bc.stateCache.NodePool()
+	db := bc.stateCache.TrieDB()
-	pool.Reference(root, common.Hash{}) // metadata reference to keep trie alive
+
-	if number := block.NumberU64(); number > 192 {
+	var (
-		if (number-192)%128 == 0 {
+		writeRetention  = uint64(128)                          // Number of trie nodes we need to retain in memory
-			pool.Commit(root, batch)
+		memoryAllowance = common.StorageSize(64 * 1024 * 1024) // Memory allowance below which to avoid writing to disk
+		timeAllowance   = 3 * time.Minute                      // Time allowance below which to avoid writing to disk
+	)
+	db.Reference(root, common.Hash{}) // metadata reference to keep trie alive
+
+	if current := block.NumberU64(); current > writeRetention {
+		// Find the next state trie we need to get rid of or commit
+		header := bc.GetHeaderByNumber(current - writeRetention)
+		chosen := header.Number.Uint64()
+
+		// Only write to disk if we exceeded our memory allowance *and* also have at
+		// least a given number of tries gapped.
+		if (db.Size() > memoryAllowance || bc.procTime > timeAllowance) && chosen >= lastWrite+writeRetention {
+			db.Commit(header.Root, batch)
+			lastWrite = chosen
+			bc.procTime = 0
+		}
+		// Garbage collect anything below our required write retention
+		db.Dereference(header.Root, common.Hash{})
+
+		if current%10000 == 0 {
+			log.Warn("Current trie pruning state", "size", db.Size(), "elapsed", bc.procTime)
 		}
-		header := bc.GetHeaderByNumber(block.NumberU64() - 192)
-		pool.Dereference(header.Root, common.Hash{})
 	}
 	if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
 		return NonStatTy, err
@@ -983,6 +1006,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			bc.reportBlock(block, receipts, err)
 			return i, events, coalescedLogs, err
 		}
+		bc.procTime += time.Since(bstart)
+
 		// Write the block to the chain and get the status.
 		status, err := bc.WriteBlockAndState(block, receipts, state)
 		if err != nil {

core/blockchain_test.go

@@ -150,7 +150,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 		blockchain.mu.Lock()
 		WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash())))
 		WriteBlock(blockchain.chainDb, block)
-		statedb.CommitTo(blockchain.chainDb, false)
+		statedb.Commit(false)
 		blockchain.mu.Unlock()
 	}
 	return nil

core/chain_makers.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // So we can deterministically seed different blockchains
@@ -162,6 +163,8 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	if config == nil {
 		config = params.TestChainConfig
 	}
+	triedb := trie.NewDatabase(db)
+
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
 		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
@@ -192,16 +195,19 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if b.engine != nil {
 			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
 			// Write state changes to db
-			_, err := statedb.CommitTo(db, config.IsEIP158(b.header.Number))
+			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
 			if err != nil {
 				panic(fmt.Sprintf("state write error: %v", err))
 			}
+			if err := triedb.Commit(root, db); err != nil {
+				panic(fmt.Sprintf("trie write error: %v", err))
+			}
 			return block, b.receipts
 		}
 		return nil, nil
 	}
 	for i := 0; i < n; i++ {
-		statedb, err := state.New(parent.Root(), state.NewDatabase(db, nil))
+		statedb, err := state.New(parent.Root(), state.NewDatabase(triedb))
 		if err != nil {
 			panic(err)
 		}

core/dao_test.go

@@ -79,7 +79,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, db); err != nil {
 			t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 		}
 		blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@@ -104,7 +104,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, db); err != nil {
 			t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 		}
 		blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@@ -130,7 +130,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, db); err != nil {
 		t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@@ -150,7 +150,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, db); err != nil {
 		t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})

core/genesis.go

@@ -34,6 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go
@@ -169,7 +170,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
 
 	// Check whether the genesis block is already written.
 	if genesis != nil {
-		block, _ := genesis.ToBlock()
+		block, _, _ := genesis.ToBlock()
 		hash := block.Hash()
 		if hash != stored {
 			return genesis.Config, block.Hash(), &GenesisMismatchError{stored, hash}
@@ -221,9 +222,10 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 }
 
 // ToBlock creates the block and state of a genesis specification.
-func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
+func (g *Genesis) ToBlock() (*types.Block, *state.StateDB, *trie.Database) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	triedb := trie.NewDatabase(diskdb)
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(triedb))
 	for addr, account := range g.Alloc {
 		statedb.AddBalance(addr, account.Balance)
 		statedb.SetCode(addr, account.Code)
@@ -252,19 +254,22 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
 	if g.Difficulty == nil {
 		head.Difficulty = params.GenesisDifficulty
 	}
-	return types.NewBlock(head, nil, nil, nil), statedb
+	return types.NewBlock(head, nil, nil, nil), statedb, triedb
 }
 
 // Commit writes the block and state of a genesis specification to the database.
 // The block is committed as the canonical head block.
 func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
-	block, statedb := g.ToBlock()
+	block, statedb, triedb := g.ToBlock()
 	if block.Number().Sign() != 0 {
 		return nil, fmt.Errorf("can't commit genesis block with number > 0")
 	}
-	if _, err := statedb.CommitTo(db, false); err != nil {
+	if _, err := statedb.Commit(false); err != nil {
 		return nil, fmt.Errorf("cannot write state: %v", err)
 	}
+	if err := triedb.Commit(block.Root(), db); err != nil {
+		return nil, err
+	}
 	if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil {
 		return nil, err
 	}

core/genesis_test.go

@@ -30,11 +30,11 @@ import (
 )
 
 func TestDefaultGenesisBlock(t *testing.T) {
-	block, _ := DefaultGenesisBlock().ToBlock()
+	block, _, _ := DefaultGenesisBlock().ToBlock()
 	if block.Hash() != params.MainnetGenesisHash {
 		t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash)
 	}
-	block, _ = DefaultTestnetGenesisBlock().ToBlock()
+	block, _, _ = DefaultTestnetGenesisBlock().ToBlock()
 	if block.Hash() != params.TestnetGenesisHash {
 		t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash)
 	}

core/state/database.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/trie"
 	lru "github.com/hashicorp/golang-lru"
 )
@@ -55,8 +54,8 @@ type Database interface {
 	// ContractCodeSize retrieves a particular contracts code's size.
 	ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
 
-	// NodePool retrieves any intermediate trie-node caching layer.
+	// TrieDB retrieves the low level trie database used for data storage.
-	NodePool() *trie.NodePool
+	TrieDB() *trie.Database
 }
 
 // Trie is a Ethereum Merkle Trie.
@@ -64,7 +63,7 @@ type Trie interface {
 	TryGet(key []byte) ([]byte, error)
 	TryUpdate(key, value []byte) error
 	TryDelete(key []byte) error
-	CommitTo(trie.DatabaseWriter) (common.Hash, error)
+	Commit(onleaf trie.LeafCallback) (common.Hash, error)
 	Hash() common.Hash
 	NodeIterator(startKey []byte) trie.NodeIterator
 	GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed
@@ -74,16 +73,18 @@ type Trie interface {
 // concurrent use and retains cached trie nodes in memory. The pool is an optional
 // intermediate trie-node memory pool between the low level storage layer and the
 // high level trie abstraction.
-func NewDatabase(db ethdb.Database, pool *trie.NodePool) Database {
+func NewDatabase(db *trie.Database) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
-	return &cachingDB{db: db, pastNodes: pool, codeSizeCache: csc}
+	return &cachingDB{
+		db:            db,
+		codeSizeCache: csc,
+	}
 }
 
 type cachingDB struct {
-	db            ethdb.Database
+	db            *trie.Database
 	mu            sync.Mutex
 	pastTries     []*trie.SecureTrie
-	pastNodes     *trie.NodePool
 	codeSizeCache *lru.Cache
 }
 
@@ -97,7 +98,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 			return cachedTrie{db.pastTries[i].Copy(), db}, nil
 		}
 	}
-	tr, err := trie.NewSecure(root, db.db, db.pastNodes, MaxTrieCacheGen)
+	tr, err := trie.NewSecure(root, db.db, MaxTrieCacheGen)
 	if err != nil {
 		return nil, err
 	}
@@ -118,7 +119,7 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
-	return trie.NewSecure(root, db.db, db.pastNodes, 0)
+	return trie.NewSecure(root, db.db, 0)
 }
 
 // CopyTrie returns an independent copy of the given trie.
@@ -135,7 +136,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 
 // ContractCode retrieves a particular contract's code.
 func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
-	code, err := db.db.Get(codeHash[:])
+	code, err := db.db.Node(codeHash)
 	if err == nil {
 		db.codeSizeCache.Add(codeHash, len(code))
 	}
@@ -154,9 +155,9 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 	return len(code), err
 }
 
-// NodePool retrieves any intermediate trie-node caching layer.
+// TrieDB retrieves any intermediate trie-node caching layer.
-func (db *cachingDB) NodePool() *trie.NodePool {
+func (db *cachingDB) TrieDB() *trie.Database {
-	return db.pastNodes
+	return db.db
 }
 
 // cachedTrie inserts its trie into a cachingDB on commit.
@@ -165,8 +166,8 @@ type cachedTrie struct {
 	db *cachingDB
 }
 
-func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
+func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
-	root, err := m.SecureTrie.CommitTo(dbw)
+	root, err := m.SecureTrie.Commit(onleaf)
 	if err == nil {
 		m.db.pushTrie(m.SecureTrie)
 	}

core/state/iterator_test.go

@@ -21,12 +21,13 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // Tests that the node iterator indeed walks over the entire database contents.
 func TestNodeIteratorCoverage(t *testing.T) {
 	// Create some arbitrary test state to iterate
-	db, mem, root, _ := makeTestState()
+	db, root, _ := makeTestState()
 
 	state, err := New(root, db)
 	if err != nil {
@@ -41,18 +42,16 @@ func TestNodeIteratorCoverage(t *testing.T) {
 	}
 	// Cross check the iterated hashes and the database/nodepool content
 	for hash := range hashes {
-		if db.NodePool().Fetch(hash) == nil {
+		if _, err := db.TrieDB().Node(hash); err != nil {
-			if _, err := mem.Get(hash.Bytes()); err != nil {
 			t.Errorf("failed to retrieve reported node %x", hash)
 		}
 	}
-	}
+	for _, hash := range db.TrieDB().Nodes() {
-	for _, hash := range db.NodePool().Nodes() {
 		if _, ok := hashes[hash]; !ok {
 			t.Errorf("state entry not reported %x", hash)
 		}
 	}
-	for _, key := range mem.Keys() {
+	for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() {
 		if bytes.HasPrefix(key, []byte("secure-key-")) {
 			continue
 		}

core/state/managed_state_test.go

@@ -27,8 +27,8 @@ import (
 var addr = common.BytesToAddress([]byte("test"))
 
 func create() (*ManagedState, *account) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	statedb, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
+	statedb, _ := New(common.Hash{}, NewDatabase(trie.NewDatabase(diskdb)))
 	ms := ManageState(statedb)
 	ms.StateDB.SetNonce(addr, 100)
 	ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr))

core/state/state_object.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie"
 )
 
 var emptyCodeHash = crypto.Keccak256(nil)
@@ -238,12 +237,12 @@ func (self *stateObject) updateRoot(db Database) {
 
 // CommitTrie the storage trie of the object to dwb.
 // This updates the trie root.
-func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error {
+func (self *stateObject) CommitTrie(db Database) error {
 	self.updateTrie(db)
 	if self.dbErr != nil {
 		return self.dbErr
 	}
-	root, err := self.trie.CommitTo(dbw)
+	root, err := self.trie.Commit(nil)
 	if err == nil {
 		self.data.Root = root
 	}

core/state/state_test.go

@@ -49,7 +49,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	// write some of them to the trie
 	s.state.updateStateObject(obj1)
 	s.state.updateStateObject(obj2)
-	s.state.CommitTo(s.db, false)
+	s.state.Commit(false)
 
 	// check that dump contains the state objects that are in trie
 	got := string(s.state.Dump())
@@ -89,7 +89,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 
 func (s *StateSuite) SetUpTest(c *checker.C) {
 	s.db, _ = ethdb.NewMemDatabase()
-	s.state, _ = New(common.Hash{}, NewDatabase(s.db, trie.NewNodePool()))
+	s.state, _ = New(common.Hash{}, NewDatabase(trie.NewDatabase(s.db)))
 }
 
 func (s *StateSuite) TestNull(c *checker.C) {
@@ -98,7 +98,7 @@ func (s *StateSuite) TestNull(c *checker.C) {
 	//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
 	var value common.Hash
 	s.state.SetState(address, common.Hash{}, value)
-	s.state.CommitTo(s.db, false)
+	s.state.Commit(false)
 	value = s.state.GetState(address, common.Hash{})
 	if !common.EmptyHash(value) {
 		c.Errorf("expected empty hash. got %x", value)
@@ -134,8 +134,9 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) {
 // use testing instead of checker because checker does not support
 // printing/logging in tests (-check.vv does not work)
 func TestSnapshot2(t *testing.T) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	state, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
+	triedb := trie.NewDatabase(diskdb)
+	state, _ := New(common.Hash{}, NewDatabase(triedb))
 
 	stateobjaddr0 := toAddr([]byte("so0"))
 	stateobjaddr1 := toAddr([]byte("so1"))
@@ -156,7 +157,7 @@ func TestSnapshot2(t *testing.T) {
 	so0.deleted = false
 	state.setStateObject(so0)
 
-	root, _ := state.CommitTo(db, false)
+	root, _ := state.Commit(false)
 	state.Reset(root)
 
 	// and one with deleted == true

core/state/statedb.go

@@ -36,6 +36,14 @@ type revision struct {
 	journalIndex int
 }
 
+var (
+	// emptyState is the known hash of an empty state trie entry.
+	emptyState = crypto.Keccak256Hash(nil)
+
+	// emptyCode is the known hash of the empty EVM bytecode.
+	emptyCode = crypto.Keccak256Hash(nil)
+)
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -568,8 +576,8 @@ func (s *StateDB) clearJournalAndRefund() {
 	s.refund = 0
 }
 
-// CommitTo writes the state to the given database.
+// Commit writes the state to the underlying in-memory trie database.
-func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root common.Hash, err error) {
+func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
 	// Commit objects to the trie.
@@ -583,13 +591,11 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
 		case isDirty:
 			// Write any contract code associated with the state object
 			if stateObject.code != nil && stateObject.dirtyCode {
-				if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil {
+				s.db.TrieDB().Insert(common.BytesToHash(stateObject.CodeHash()), stateObject.code)
-					return common.Hash{}, err
-				}
 				stateObject.dirtyCode = false
 			}
 			// Write any storage changes in the state object to its storage trie.
-			if err := stateObject.CommitTrie(s.db, dbw); err != nil {
+			if err := stateObject.CommitTrie(s.db); err != nil {
 				return common.Hash{}, err
 			}
 			// Update the object in the main account trie.
@@ -598,7 +604,20 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
 		delete(s.stateObjectsDirty, addr)
 	}
 	// Write trie changes.
-	root, err = s.trie.CommitTo(dbw)
+	root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error {
+		var account Account
+		if err := rlp.DecodeBytes(leaf, &account); err != nil {
+			return nil
+		}
+		if account.Root != emptyState {
+			s.db.TrieDB().Reference(account.Root, parent)
+		}
+		code := common.BytesToHash(account.CodeHash)
+		if code != emptyCode {
+			s.db.TrieDB().Reference(code, parent)
+		}
+		return nil
+	})
 	log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads())
 	return root, err
 }

core/state/statedb_test.go

@@ -41,7 +41,7 @@ import (
 func TestUpdateLeaks(t *testing.T) {
 	// Create an empty state database
 	db, _ := ethdb.NewMemDatabase()
-	state, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
+	state, _ := New(common.Hash{}, NewDatabase(trie.NewDatabase(db)))
 
 	// Update it with some accounts
 	for i := byte(0); i < 255; i++ {
@@ -69,8 +69,8 @@ func TestIntermediateLeaks(t *testing.T) {
 	// Create two state databases, one transitioning to the final state, the other final from the beginning
 	transDb, _ := ethdb.NewMemDatabase()
 	finalDb, _ := ethdb.NewMemDatabase()
-	transState, _ := New(common.Hash{}, NewDatabase(transDb, trie.NewNodePool()))
+	transState, _ := New(common.Hash{}, NewDatabase(trie.NewDatabase(transDb)))
-	finalState, _ := New(common.Hash{}, NewDatabase(finalDb, trie.NewNodePool()))
+	finalState, _ := New(common.Hash{}, NewDatabase(trie.NewDatabase(finalDb)))
 
 	modify := func(state *StateDB, addr common.Address, i, tweak byte) {
 		state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@@ -98,10 +98,10 @@ func TestIntermediateLeaks(t *testing.T) {
 	}
 
 	// Commit and cross check the databases.
-	if _, err := transState.CommitTo(transDb, false); err != nil {
+	if _, err := transState.Commit(false); err != nil {
 		t.Fatalf("failed to commit transition state: %v", err)
 	}
-	if _, err := finalState.CommitTo(finalDb, false); err != nil {
+	if _, err := finalState.Commit(false); err != nil {
 		t.Fatalf("failed to commit final state: %v", err)
 	}
 	for _, key := range finalDb.Keys() {
@@ -123,8 +123,8 @@ func TestIntermediateLeaks(t *testing.T) {
 // https://github.com/ethereum/go-ethereum/pull/15549.
 func TestCopy(t *testing.T) {
 	// Create a random state test to copy and modify "independently"
-	mem, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	orig, _ := New(common.Hash{}, NewDatabase(mem, trie.NewNodePool()))
+	orig, _ := New(common.Hash{}, NewDatabase(trie.NewDatabase(diskdb)))
 
 	for i := byte(0); i < 255; i++ {
 		obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@@ -335,9 +335,9 @@ func (test *snapshotTest) String() string {
 func (test *snapshotTest) run() bool {
 	// Run all actions and create snapshots.
 	var (
-		db, _        = ethdb.NewMemDatabase()
+		diskdb, _    = ethdb.NewMemDatabase()
-		mp           = trie.NewNodePool()
+		triedb       = trie.NewDatabase(diskdb)
-		state, _     = New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
+		state, _     = New(common.Hash{}, NewDatabase(triedb))
 		snapshotRevs = make([]int, len(test.snapshots))
 		sindex       = 0
 	)
@@ -352,7 +352,7 @@ func (test *snapshotTest) run() bool {
 	// Revert all snapshots in reverse order. Each revert must yield a state
 	// that is equivalent to fresh state with all actions up the snapshot applied.
 	for sindex--; sindex >= 0; sindex-- {
-		checkstate, _ := New(common.Hash{}, NewDatabase(db, mp))
+		checkstate, _ := New(common.Hash{}, NewDatabase(triedb))
 		for _, action := range test.actions[:test.snapshots[sindex]] {
 			action.fn(action, checkstate)
 		}
@@ -411,7 +411,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error {
 
 func (s *StateSuite) TestTouchDelete(c *check.C) {
 	s.state.GetOrNewStateObject(common.Address{})
-	root, _ := s.state.CommitTo(s.db, false)
+	root, _ := s.state.Commit(false)
 	s.state.Reset(root)
 
 	snapshot := s.state.Snapshot()
@@ -419,7 +419,6 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 	if len(s.state.stateObjectsDirty) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
-
 	s.state.RevertToSnapshot(snapshot)
 	if len(s.state.stateObjectsDirty) != 0 {
 		c.Fatal("expected no dirty state object")

core/state/sync_test.go

@@ -36,10 +36,10 @@ type testAccount struct {
 }
 
 // makeTestState create a sample test state to test node-wise reconstruction.
-func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) {
+func makeTestState() (Database, common.Hash, []*testAccount) {
 	// Create an empty state
-	mem, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	db := NewDatabase(mem, trie.NewNodePool())
+	db := NewDatabase(trie.NewDatabase(diskdb))
 	state, _ := New(common.Hash{}, db)
 
 	// Fill it with some arbitrary data
@@ -61,17 +61,17 @@ func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount)
 		state.updateStateObject(obj)
 		accounts = append(accounts, acc)
 	}
-	root, _ := state.CommitTo(mem, false)
+	root, _ := state.Commit(false)
 
 	// Return the generated state
-	return db, mem, root, accounts
+	return db, root, accounts
 }
 
 // checkStateAccounts cross references a reconstructed state with an expected
 // account array.
 func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
 	// Check root availability and state contents
-	state, err := New(root, NewDatabase(db, trie.NewNodePool()))
+	state, err := New(root, NewDatabase(trie.NewDatabase(db)))
 	if err != nil {
 		t.Fatalf("failed to create state trie at %x: %v", root, err)
 	}
@@ -96,7 +96,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
 	if v, _ := db.Get(root[:]); v == nil {
 		return nil // Consider a non existent state consistent.
 	}
-	trie, err := trie.New(root, db, trie.NewNodePool())
+	trie, err := trie.New(root, trie.NewDatabase(db))
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
 	if _, err := db.Get(root.Bytes()); err != nil {
 		return nil // Consider a non existent state consistent.
 	}
-	state, err := New(root, NewDatabase(db, trie.NewNodePool()))
+	state, err := New(root, NewDatabase(trie.NewDatabase(db)))
 	if err != nil {
 		return err
 	}
@@ -138,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T)    { testIterativeStateSync(t,
 
 func testIterativeStateSync(t *testing.T, batch int) {
 	// Create a random state to copy
-	srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -148,15 +148,10 @@ func testIterativeStateSync(t *testing.T, batch int) {
 	for len(queue) > 0 {
 		results := make([]trie.SyncResult, len(queue))
 		for i, hash := range queue {
-			var (
+			data, err := srcDb.TrieDB().Node(hash)
-				data = srcDb.NodePool().Fetch(hash)
+			if err != nil {
-				err  error
-			)
-			if data == nil {
-				if data, err = srcMem.Get(hash.Bytes()); err != nil {
 				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
-			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}
 		if _, index, err := sched.Process(results); err != nil {
@@ -175,7 +170,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
 // partial results are returned, and the others sent only later.
 func TestIterativeDelayedStateSync(t *testing.T) {
 	// Create a random state to copy
-	srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -186,15 +181,10 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 		// Sync only half of the scheduled nodes
 		results := make([]trie.SyncResult, len(queue)/2+1)
 		for i, hash := range queue[:len(results)] {
-			var (
+			data, err := srcDb.TrieDB().Node(hash)
-				data = srcDb.NodePool().Fetch(hash)
+			if err != nil {
-				err  error
-			)
-			if data == nil {
-				if data, err = srcMem.Get(hash.Bytes()); err != nil {
 				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
-			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}
 		if _, index, err := sched.Process(results); err != nil {
@@ -217,7 +207,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T)    { testIterativeRandomS
 
 func testIterativeRandomStateSync(t *testing.T, batch int) {
 	// Create a random state to copy
-	srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -231,15 +221,10 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 		// Fetch all the queued nodes in a random order
 		results := make([]trie.SyncResult, 0, len(queue))
 		for hash := range queue {
-			var (
+			data, err := srcDb.TrieDB().Node(hash)
-				data = srcDb.NodePool().Fetch(hash)
+			if err != nil {
-				err  error
-			)
-			if data == nil {
-				if data, err = srcMem.Get(hash.Bytes()); err != nil {
 				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
-			}
 			results = append(results, trie.SyncResult{Hash: hash, Data: data})
 		}
 		// Feed the retrieved results back and queue new tasks
@@ -262,7 +247,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 // partial results are returned (Even those randomly), others sent only later.
 func TestIterativeRandomDelayedStateSync(t *testing.T) {
 	// Create a random state to copy
-	srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -278,15 +263,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 		for hash := range queue {
 			delete(queue, hash)
 
-			var (
+			data, err := srcDb.TrieDB().Node(hash)
-				data = srcDb.NodePool().Fetch(hash)
+			if err != nil {
-				err  error
-			)
-			if data == nil {
-				if data, err = srcMem.Get(hash.Bytes()); err != nil {
 				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
-			}
 			results = append(results, trie.SyncResult{Hash: hash, Data: data})
 
 			if len(results) >= cap(results) {
@@ -312,9 +292,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 // the database.
 func TestIncompleteStateSync(t *testing.T) {
 	// Create a random state to copy
-	srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
-	checkTrieConsistency(srcMem, srcRoot)
+	checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot)
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -326,15 +306,10 @@ func TestIncompleteStateSync(t *testing.T) {
 		// Fetch a batch of state nodes
 		results := make([]trie.SyncResult, len(queue))
 		for i, hash := range queue {
-			var (
+			data, err := srcDb.TrieDB().Node(hash)
-				data = srcDb.NodePool().Fetch(hash)
+			if err != nil {
-				err  error
-			)
-			if data == nil {
-				if data, err = srcMem.Get(hash.Bytes()); err != nil {
 				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
-			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}
 		// Process each of the state nodes

core/tx_pool_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // testTxPoolConfig is a transaction pool configuration without stateful disk
@@ -78,8 +79,8 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec
 }
 
 func setupTxPool() (*TxPool, *ecdsa.PrivateKey) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(diskdb)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	key, _ := crypto.GenerateKey()
@@ -159,7 +160,7 @@ func (c *testChain) State() (*state.StateDB, error) {
 	stdb := c.statedb
 	if *c.trigger {
 		db, _ := ethdb.NewMemDatabase()
-		c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 		// simulate that the new head block included tx0 and tx1
 		c.statedb.SetNonce(c.address, 2)
 		c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether))
@@ -178,7 +179,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
 		db, _      = ethdb.NewMemDatabase()
 		key, _     = crypto.GenerateKey()
 		address    = crypto.PubkeyToAddress(key.PublicKey)
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 		trigger    = false
 	)
 
@@ -338,7 +339,7 @@ func TestTransactionChainFork(t *testing.T) {
 	addr := crypto.PubkeyToAddress(key.PublicKey)
 	resetState := func() {
 		db, _ := ethdb.NewMemDatabase()
-		statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+		statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 		statedb.AddBalance(addr, big.NewInt(100000000000000))
 
 		pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@@ -368,7 +369,7 @@ func TestTransactionDoubleNonce(t *testing.T) {
 	addr := crypto.PubkeyToAddress(key.PublicKey)
 	resetState := func() {
 		db, _ := ethdb.NewMemDatabase()
-		statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+		statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 		statedb.AddBalance(addr, big.NewInt(100000000000000))
 
 		pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@@ -737,7 +738,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) {
 
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -826,7 +827,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) {
 
 	// Create the pool to test the non-expiration enforcement
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -981,7 +982,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) {
 
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1028,7 +1029,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) {
 
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1063,7 +1064,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) {
 
 	// Create the pool to test the limit enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1112,7 +1113,7 @@ func TestTransactionPoolRepricing(t *testing.T) {
 
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1211,7 +1212,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
 
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1274,7 +1275,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1376,7 +1377,7 @@ func TestTransactionReplacement(t *testing.T) {
 
 	// Create the pool to test the pricing enforcement with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1471,7 +1472,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
 
 	// Create the original pool to inject transaction into the journal
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1570,7 +1571,7 @@ func TestTransactionStatusCheck(t *testing.T) {
 
 	// Create the pool to test the status retrievals with
 	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)

core/vm/runtime/runtime.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // Config is a basic type specifying certain configuration flags for running
@@ -102,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 
 	if cfg.State == nil {
 		db, _ := ethdb.NewMemDatabase()
-		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	}
 	var (
 		address = common.StringToAddress("contract")
@@ -133,7 +134,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) {
 
 	if cfg.State == nil {
 		db, _ := ethdb.NewMemDatabase()
-		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	}
 	var (
 		vmenv  = NewEnv(cfg)

core/vm/runtime/runtime_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 func TestDefaults(t *testing.T) {
@@ -95,7 +96,7 @@ func TestExecute(t *testing.T) {
 
 func TestCall(t *testing.T) {
 	db, _ := ethdb.NewMemDatabase()
-	state, _ := state.New(common.Hash{}, state.NewDatabase(db, nil))
+	state, _ := state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 	address := common.HexToAddress("0x0a")
 	state.SetCode(address, []byte{
 		byte(vm.PUSH1), 10,

eth/api.go

@@ -462,11 +462,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
 
-	oldTrie, err := trie.NewSecure(startBlock.Root(), api.eth.chainDb, nil, 0)
+	oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
 	if err != nil {
 		return nil, err
 	}
-	newTrie, err := trie.NewSecure(endBlock.Root(), api.eth.chainDb, nil, 0)
+	newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
 	if err != nil {
 		return nil, err
 	}

eth/api_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 var dumper = spew.ConfigState{Indent: "    "}
@@ -32,7 +33,7 @@ func TestStorageRangeAt(t *testing.T) {
 	// Create a state where account 0x010000... has a few storage entries.
 	var (
 		db, _    = ethdb.NewMemDatabase()
-		state, _ = state.New(common.Hash{}, state.NewDatabase(db, nil))
+		state, _ = state.New(common.Hash{}, state.NewDatabase(trie.NewDatabase(db)))
 		addr     = common.Address{0x01}
 		keys     = []common.Hash{ // hashes of Keys of storage
 			common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"),

eth/api_tracer.go

@@ -200,7 +200,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			return nil, fmt.Errorf("parent block #%d not found", number-1)
 		}
 	}
-	statedb, err := state.New(start.Root(), state.NewDatabase(db, nil))
+	statedb, err := state.New(start.Root(), state.NewDatabase(trie.NewDatabase(db)))
 	if err != nil {
 		// If the starting state is missing, allow some number of blocks to be reexecuted
 		reexec := defaultTraceReexec
@@ -213,7 +213,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			if start == nil {
 				break
 			}
-			if statedb, err = state.New(start.Root(), state.NewDatabase(db, nil)); err == nil {
+			if statedb, err = state.New(start.Root(), state.NewDatabase(trie.NewDatabase(db))); err == nil {
 				break
 			}
 		}
@@ -340,7 +340,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				break
 			}
 			// Finalize the state so any modifications are written to the trie
-			root, err := statedb.CommitTo(db, true)
+			root, err := statedb.Commit(true)
 			if err != nil {
 				failed = err
 				break
@@ -367,7 +367,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				db.Prune(root)
 				log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(start))
 
-				statedb, _ = state.New(root, state.NewDatabase(db, nil))
+				statedb, _ = state.New(root, state.NewDatabase(trie.NewDatabase(db)))
 			}
 		}
 	}()
@@ -555,7 +555,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 		if block == nil {
 			break
 		}
-		if statedb, err = state.New(block.Root(), state.NewDatabase(db, nil)); err == nil {
+		if statedb, err = state.New(block.Root(), state.NewDatabase(trie.NewDatabase(db))); err == nil {
 			break
 		}
 	}
@@ -587,7 +587,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 			return nil, err
 		}
 		// Finalize the state so any modifications are written to the trie
-		root, err := statedb.CommitTo(db, true)
+		root, err := statedb.Commit(true)
 		if err != nil {
 			return nil, err
 		}
@@ -603,7 +603,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 			db.Prune(root)
 			log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(begin))
 
-			statedb, _ = state.New(root, state.NewDatabase(db, nil))
+			statedb, _ = state.New(root, state.NewDatabase(trie.NewDatabase(db)))
 		}
 	}
 	log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start))

eth/downloader/downloader_test.go

@@ -293,7 +293,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
 func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
 	// For now only check that the state trie is correct
 	if block := dl.GetBlockByHash(hash); block != nil {
-		_, err := trie.NewSecure(block.Root(), dl.stateDb, nil, 0) // nil trie node-cache, ensure we have everything on disk
+		_, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb), 0)
 		return err
 	}
 	return fmt.Errorf("non existent block: %x", hash[:4])
@@ -660,7 +660,7 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
 			index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
 		}
 		if index > 0 {
-			if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb, nil)); statedb == nil || err != nil {
+			if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil {
 				t.Fatalf("state reconstruction failed: %v", err)
 			}
 		}

eth/handler_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // Tests that protocol versions and modes of operations are matched up properly.
@@ -372,7 +373,7 @@ func testGetNodeData(t *testing.T, protocol int) {
 	}
 	accounts := []common.Address{testBank, acc1Addr, acc2Addr}
 	for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ {
-		trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), state.NewDatabase(statedb, nil))
+		trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), state.NewDatabase(trie.NewDatabase(statedb)))
 
 		for j, acc := range accounts {
 			state, _ := pm.blockchain.State()

les/handler.go

@@ -579,7 +579,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 		for _, req := range req.Reqs {
 			// Retrieve the requested state entry, stopping if enough was found
 			if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil {
-				if trie, _ := trie.New(header.Root, pm.chainDb, nil); trie != nil {
+				if trie, _ := trie.New(header.Root, trie.NewDatabase(pm.chainDb)); trie != nil {
 					sdata := trie.Get(req.AccKey)
 					var acc state.Account
 					if err := rlp.DecodeBytes(sdata, &acc); err == nil {
@@ -706,13 +706,13 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			}
 			// Retrieve the requested state entry, stopping if enough was found
 			if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil {
-				if tr, _ := trie.New(header.Root, pm.chainDb, nil); tr != nil {
+				if tr, _ := trie.New(header.Root, trie.NewDatabase(pm.chainDb)); tr != nil {
 					if len(req.AccKey) > 0 {
 						sdata := tr.Get(req.AccKey)
 						tr = nil
 						var acc state.Account
 						if err := rlp.DecodeBytes(sdata, &acc); err == nil {
-							tr, _ = trie.New(acc.Root, pm.chainDb, nil)
+							tr, _ = trie.New(acc.Root, trie.NewDatabase(pm.chainDb))
 						}
 					}
 					if tr != nil {
@@ -757,7 +757,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			}
 			if tr == nil || req.BHash != lastBHash {
 				if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil {
-					tr, _ = trie.New(header.Root, pm.chainDb, nil)
+					tr, _ = trie.New(header.Root, trie.NewDatabase(pm.chainDb))
 				} else {
 					tr = nil
 				}
@@ -771,7 +771,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						str = nil
 						var acc state.Account
 						if err := rlp.DecodeBytes(sdata, &acc); err == nil {
-							str, _ = trie.New(acc.Root, pm.chainDb, nil)
+							str, _ = trie.New(acc.Root, trie.NewDatabase(pm.chainDb))
 						}
 						lastAccKey = common.CopyBytes(req.AccKey)
 					}
@@ -858,7 +858,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
 				sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1)
 				if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) {
-					if tr, _ := trie.New(root, trieDb, nil); tr != nil {
+					if tr, _ := trie.New(root, trie.NewDatabase(trieDb)); tr != nil {
 						var encNumber [8]byte
 						binary.BigEndian.PutUint64(encNumber[:], req.BlockNum)
 						var proof light.NodeList
@@ -910,7 +910,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 				var prefix string
 				root, prefix = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx)
 				if root != (common.Hash{}) {
-					if t, err := trie.New(root, ethdb.NewTable(pm.chainDb, prefix), nil); err == nil {
+					if t, err := trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))); err == nil {
 						tr = t
 					}
 				}

les/handler_test.go

@@ -359,7 +359,7 @@ func testGetProofs(t *testing.T, protocol int) {
 	for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ {
 		header := bc.GetHeaderByNumber(i)
 		root := header.Root
-		trie, _ := trie.New(root, db, nil)
+		trie, _ := trie.New(root, trie.NewDatabase(db))
 
 		for _, acc := range accounts {
 			req := ProofReq{

les/odr_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/light"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 type odrTestFn func(ctx context.Context, db ethdb.Database, config *params.ChainConfig, bc *core.BlockChain, lc *light.LightChain, bhash common.Hash) []byte
@@ -90,7 +91,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon
 	for _, addr := range acc {
 		if bc != nil {
 			header := bc.GetHeaderByHash(bhash)
-			st, err = state.New(header.Root, state.NewDatabase(db, nil))
+			st, err = state.New(header.Root, state.NewDatabase(trie.NewDatabase(db)))
 		} else {
 			header := lc.GetHeaderByHash(bhash)
 			st = light.NewState(ctx, header, lc.Odr())
@@ -123,7 +124,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai
 		data[35] = byte(i)
 		if bc != nil {
 			header := bc.GetHeaderByHash(bhash)
-			statedb, err := state.New(header.Root, state.NewDatabase(db, nil))
+			statedb, err := state.New(header.Root, state.NewDatabase(trie.NewDatabase(db)))
 
 			if err == nil {
 				from := statedb.GetOrNewStateObject(testBankAddress)

light/nodeset.go

@@ -99,7 +99,7 @@ func (db *NodeSet) NodeList() NodeList {
 }
 
 // Store writes the contents of the set to the given database
-func (db *NodeSet) Store(target trie.Database) {
+func (db *NodeSet) Store(target trie.DatabaseWriter) {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
@@ -112,7 +112,7 @@ func (db *NodeSet) Store(target trie.Database) {
 type NodeList []rlp.RawValue
 
 // Store writes the contents of the list to the given database
-func (n NodeList) Store(db trie.Database) {
+func (n NodeList) Store(db trie.DatabaseWriter) {
 	for _, node := range n {
 		db.Put(crypto.Keccak256(node), node)
 	}

light/odr_test.go

@@ -74,7 +74,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
 	case *ReceiptsRequest:
 		req.Receipts = core.GetBlockReceipts(odr.sdb, req.Hash, core.GetBlockNumber(odr.sdb, req.Hash))
 	case *TrieRequest:
-		t, _ := trie.New(req.Id.Root, odr.sdb, nil)
+		t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb))
 		nodes := NewNodeSet()
 		t.Prove(req.Key, 0, nodes)
 		req.Proof = nodes
@@ -131,7 +131,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc
 		st = NewState(ctx, header, lc.Odr())
 	} else {
 		header := bc.GetHeaderByHash(bhash)
-		st, _ = state.New(header.Root, state.NewDatabase(db, nil))
+		st, _ = state.New(header.Root, state.NewDatabase(trie.NewDatabase(db)))
 	}
 
 	var res []byte
@@ -171,7 +171,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain
 		} else {
 			chain = bc
 			header = bc.GetHeaderByHash(bhash)
-			st, _ = state.New(header.Root, state.NewDatabase(db, nil))
+			st, _ = state.New(header.Root, state.NewDatabase(trie.NewDatabase(db)))
 		}
 
 		// Perform read-only call.

light/postprocess.go

@@ -141,7 +141,7 @@ func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) e
 		root = GetChtRoot(c.db, section-1, lastSectionHead)
 	}
 	var err error
-	c.trie, err = trie.New(root, c.cdb, nil)
+	c.trie, err = trie.New(root, trie.NewDatabase(c.cdb))
 	c.section = section
 	return err
 }
@@ -163,17 +163,14 @@ func (c *ChtIndexerBackend) Process(header *types.Header) {
 
 // Commit implements core.ChainIndexerBackend
 func (c *ChtIndexerBackend) Commit() error {
-	batch := c.cdb.NewBatch()
+	root, err := c.trie.Commit(nil)
-	root, err := c.trie.CommitTo(batch)
 	if err != nil {
 		return err
-	} else {
+	}
-		batch.Write()
 	if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 {
 		log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root))
 	}
 	StoreChtRoot(c.db, c.section, c.lastHash, root)
-	}
 	return nil
 }
 
@@ -236,7 +233,7 @@ func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.H
 		root = GetBloomTrieRoot(b.db, section-1, lastSectionHead)
 	}
 	var err error
-	b.trie, err = trie.New(root, b.cdb, nil)
+	b.trie, err = trie.New(root, trie.NewDatabase(b.cdb))
 	b.section = section
 	return err
 }
@@ -279,17 +276,13 @@ func (b *BloomTrieIndexerBackend) Commit() error {
 			b.trie.Delete(encKey[:])
 		}
 	}
-
+	root, err := b.trie.Commit(nil)
-	batch := b.cdb.NewBatch()
-	root, err := b.trie.CommitTo(batch)
 	if err != nil {
 		return err
-	} else {
+	}
-		batch.Write()
 	sectionHead := b.sectionHeads[b.bloomTrieRatio-1]
 	log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize))
 	StoreBloomTrieRoot(b.db, b.section, sectionHead, root)
-	}
 
 	return nil
 }

light/trie.go

@@ -83,7 +83,7 @@ func (db *odrDatabase) ContractCodeSize(addrHash, codeHash common.Hash) (int, er
 	return len(code), err
 }
 
-func (db *odrDatabase) NodePool() *trie.NodePool {
+func (db *odrDatabase) TrieDB() *trie.Database {
 	return nil
 }
 
@@ -117,11 +117,11 @@ func (t *odrTrie) TryDelete(key []byte) error {
 	})
 }
 
-func (t *odrTrie) CommitTo(db trie.DatabaseWriter) (common.Hash, error) {
+func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
 	if t.trie == nil {
 		return t.id.Root, nil
 	}
-	return t.trie.CommitTo(db)
+	return t.trie.Commit(onleaf)
 }
 
 func (t *odrTrie) Hash() common.Hash {
@@ -145,7 +145,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
 	for {
 		var err error
 		if t.trie == nil {
-			t.trie, err = trie.New(t.id.Root, t.db.backend.Database(), nil)
+			t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database()))
 		}
 		if err == nil {
 			err = fn()
@@ -171,7 +171,7 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator {
 	// Open the actual non-ODR trie if that hasn't happened yet.
 	if t.trie == nil {
 		it.do(func() error {
-			t, err := trie.New(t.id.Root, t.db.backend.Database(), nil)
+			t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database()))
 			if err == nil {
 				it.t.trie = t
 			}

light/trie_test.go

@@ -50,7 +50,7 @@ func TestNodeIterator(t *testing.T) {
 	odr := &testOdr{sdb: fulldb, ldb: lightdb}
 	head := blockchain.CurrentHeader()
 	lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root)
-	fullTrie, _ := state.NewDatabase(fulldb, nil).OpenTrie(head.Root)
+	fullTrie, _ := state.NewDatabase(trie.NewDatabase(fulldb)).OpenTrie(head.Root)
 	if err := diffTries(fullTrie, lightTrie); err != nil {
 		t.Fatal(err)
 	}

tests/state_test_util.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 // StateTest checks transaction processing without block context.
@@ -125,7 +126,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD
 	if !ok {
 		return nil, UnsupportedForkError{subtest.Fork}
 	}
-	block, _ := t.genesis(config).ToBlock()
+	block, _, _ := t.genesis(config).ToBlock()
 	db, _ := ethdb.NewMemDatabase()
 	statedb := MakePreState(db, t.json.Pre)
 
@@ -147,7 +148,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD
 	if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) {
 		return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
 	}
-	root, _ := statedb.CommitTo(db, config.IsEIP158(block.Number()))
+	root, _ := statedb.Commit(config.IsEIP158(block.Number()))
 	if root != common.Hash(post.Root) {
 		return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
 	}
@@ -159,7 +160,7 @@ func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
 }
 
 func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB {
-	sdb := state.NewDatabase(db, nil)
+	sdb := state.NewDatabase(trie.NewDatabase(db))
 	statedb, _ := state.New(common.Hash{}, sdb)
 	for addr, a := range accounts {
 		statedb.SetCode(addr, a.Code)
@@ -170,7 +171,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB
 		}
 	}
 	// Commit and re-open to start with a clean state.
-	root, _ := statedb.CommitTo(db, false)
+	root, _ := statedb.Commit(false)
 	statedb, _ = state.New(root, sdb)
 	return statedb
 }

trie/database.go

@@ -0,0 +1,304 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// secureKeyPrefix is the database key prefix used to store trie node preimages.
+var secureKeyPrefix = []byte("secure-key-")
+
+// secureKeyLength is the length of the above prefix + 32byte hash.
+const secureKeyLength = 11 + 32
+
+// DatabaseReader wraps the Get and Has method of a backing store for the trie.
+type DatabaseReader interface {
+	// Get retrieves the value associated with key form the database.
+	Get(key []byte) (value []byte, err error)
+
+	// Has retrieves whether a key is present in the database.
+	Has(key []byte) (bool, error)
+}
+
+// DatabaseWriter wraps the Put method of a backing store for the trie.
+type DatabaseWriter interface {
+	// Put stores the mapping key->value in the database. Implementations must not
+	// hold onto the value as the trie will reuse the slice across calls to Put.
+	Put(key, value []byte) error
+}
+
+// Database is an intermediate write layer between the trie data structures and
+// the disk database. The aim is to accumulate trie writes in-memory and only
+// periodically flush a couple tries to disk, garbage collecting the remainder.
+type Database struct {
+	diskdb DatabaseReader // Persistent storage for matured trie nodes
+
+	nodes    map[common.Hash][]byte                   // Cached data blocks of the trie nodes
+	parents  map[common.Hash]int                      // Number of live nodes referencing a given one
+	children map[common.Hash]map[common.Hash]struct{} // Set of children referenced by given nodes
+
+	preimages map[common.Hash][]byte // Preimages of nodes from the secure trie
+	seckeybuf [secureKeyLength]byte  // Ephemeral buffer for calculating preimage keys
+
+	gctime  time.Duration      // Time spent on garbage collection since last commit
+	gcnodes uint64             // Nodes garbage collected since last commit
+	gcsize  common.StorageSize // Data storage garbage collected since last commit
+
+	size common.StorageSize // Storage size of the memory cache
+	lock sync.RWMutex
+}
+
+// NewDatabase creates a new trie database to store ephemeral trie content before
+// its written out to disk or garbage collected.
+func NewDatabase(diskdb DatabaseReader) *Database {
+	db := &Database{
+		diskdb:    diskdb,
+		nodes:     make(map[common.Hash][]byte),
+		parents:   make(map[common.Hash]int),
+		children:  make(map[common.Hash]map[common.Hash]struct{}),
+		preimages: make(map[common.Hash][]byte),
+	}
+	db.children[common.Hash{}] = make(map[common.Hash]struct{})
+	return db
+}
+
+// DiskDB retrieves the persistent storage backing the trie database.
+func (db *Database) DiskDB() DatabaseReader {
+	return db.diskdb
+}
+
+// Insert writes a new trie node to the memory database if it's yet unknown. The
+// method will make a copy of the slice.
+func (db *Database) Insert(hash common.Hash, blob []byte) {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	db.insert(hash, blob)
+}
+
+// insert is the private locked version of Insert.
+func (db *Database) insert(hash common.Hash, blob []byte) {
+	if _, ok := db.nodes[hash]; ok {
+		return
+	}
+	db.nodes[hash] = common.CopyBytes(blob)
+	db.children[hash] = make(map[common.Hash]struct{})
+
+	db.size += common.StorageSize(common.HashLength + len(blob))
+}
+
+// insertPreimage writes a new trie node pre-image to the memory database if it's
+// yet unknown. The method will make a copy of the slice.
+//
+// Note, this method assumes that the database's lock is held!
+func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
+	if _, ok := db.preimages[hash]; ok {
+		return
+	}
+	db.preimages[hash] = common.CopyBytes(preimage)
+	db.size += common.StorageSize(common.HashLength + len(preimage))
+}
+
+// Node retrieves a cached trie node from memory. If it cannot be found cached,
+// the method queries the persistent database for the content.
+func (db *Database) Node(hash common.Hash) ([]byte, error) {
+	// Retrieve the node from cache if available
+	db.lock.RLock()
+	blob := db.nodes[hash]
+	db.lock.RUnlock()
+
+	if blob != nil {
+		return blob, nil
+	}
+	// Content unavailable in memory, attempt to retrieve from disk
+	return db.diskdb.Get(hash[:])
+}
+
+// preimage retrieves a cached trie node pre-image from memory. If it cannot be
+// found cached, the method queries the persistent database for the content.
+func (db *Database) preimage(hash common.Hash) ([]byte, error) {
+	// Retrieve the node from cache if available
+	db.lock.RLock()
+	preimage := db.preimages[hash]
+	db.lock.RUnlock()
+
+	if preimage != nil {
+		return preimage, nil
+	}
+	// Content unavailable in memory, attempt to retrieve from disk
+	return db.diskdb.Get(db.secureKey(hash[:]))
+}
+
+// secureKey returns the database key for the preimage of key, as an ephemeral
+// buffer. The caller must not hold onto the return value because it will become
+// invalid on the next call.
+func (db *Database) secureKey(key []byte) []byte {
+	buf := append(db.seckeybuf[:0], secureKeyPrefix...)
+	buf = append(buf, key...)
+	return buf
+}
+
+// Nodes retrieves the hashes of all the nodes cached within the memory database.
+// This method is extremely expensive and should only be used to validate internal
+// states in test code.
+func (db *Database) Nodes() []common.Hash {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	var hashes = make([]common.Hash, 0, len(db.nodes))
+	for hash := range db.nodes {
+		hashes = append(hashes, hash)
+	}
+	return hashes
+}
+
+// Preimages retrieves the hashes of all the node pre-images cached within the
+// memory database. This method is extremely expensive and should only be used
+// to validate internal states in test code.
+func (db *Database) Preimages() []common.Hash {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	var hashes = make([]common.Hash, 0, len(db.nodes))
+	for hash := range db.preimages {
+		hashes = append(hashes, hash)
+	}
+	return hashes
+}
+
+// Reference adds a new reference from a parent node to a child node.
+func (db *Database) Reference(child common.Hash, parent common.Hash) {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	db.reference(child, parent)
+}
+
+// reference is the private locked version of Reference.
+func (db *Database) reference(child common.Hash, parent common.Hash) {
+	// If the node does not exist, it's a node pulled from disk, skip
+	if _, ok := db.nodes[child]; !ok {
+		return
+	}
+	db.parents[child]++
+	db.children[parent][child] = struct{}{}
+}
+
+// Dereference removes an existing reference from a parent node to a child node.
+func (db *Database) Dereference(child common.Hash, parent common.Hash) {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	nodes, storage, start := len(db.nodes), db.size, time.Now()
+	db.dereference(child, parent)
+
+	db.gcnodes += uint64(nodes - len(db.nodes))
+	db.gcsize += storage - db.size
+	db.gctime += time.Since(start)
+}
+
+// dereference is the private locked version of Dereference.
+func (db *Database) dereference(child common.Hash, parent common.Hash) {
+	// If the node does not exist, it's a previously comitted node.
+	blob, ok := db.nodes[child]
+	if !ok {
+		return
+	}
+	delete(db.children[parent], child)
+	db.parents[child]--
+
+	// If there are no more references to the child, delete it and cascade
+	if db.parents[child] == 0 {
+		for child := range db.children[child] {
+			db.dereference(child, child)
+		}
+		delete(db.nodes, child)
+		delete(db.parents, child)
+		delete(db.children, child)
+
+		db.size -= common.StorageSize(common.HashLength + len(blob))
+	}
+}
+
+// Commit iterates over all the children of a particular node, writes them out
+// to disk, forcefully tearing down all references in both directions.
+//
+// As a side effect, all pre-images accumulated up to this point are also written.
+func (db *Database) Commit(node common.Hash, writer DatabaseWriter) error {
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	// Write out all the accumulated trie node preimages
+	for hash, preimage := range db.preimages {
+		if err := writer.Put(db.secureKey(hash[:]), preimage); err != nil {
+			log.Error("Failed to commit preimage from mempool", "err", err)
+			return err
+		}
+		db.size -= common.StorageSize(common.HashLength + len(preimage))
+	}
+	db.preimages = make(map[common.Hash][]byte)
+
+	// Write out the trie itself and dereference any flushed content
+	nodes, storage, start := len(db.nodes), db.size, time.Now()
+	if err := db.commit(node, writer); err != nil {
+		log.Error("Failed to commit trie from mempool", "err", err)
+		return err
+	}
+	log.Debug("Persistend trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.size, "time", time.Since(start),
+		"gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.size)
+
+	// Reset the garbage collection statistics
+	db.gcnodes, db.gcsize, db.gctime = 0, 0, 0
+	return nil
+}
+
+// commit is the private locked version of Commit.
+func (db *Database) commit(node common.Hash, writer DatabaseWriter) error {
+	// If the node does not exist, it's a previously comitted node.
+	blob, ok := db.nodes[node]
+	if !ok {
+		return nil
+	}
+	for child := range db.children[node] {
+		if err := db.commit(child, writer); err != nil {
+			return err
+		}
+	}
+	if err := writer.Put(node[:], blob); err != nil {
+		return err
+	}
+	delete(db.nodes, node)
+	delete(db.parents, node)
+	delete(db.children, node)
+
+	db.size -= common.StorageSize(common.HashLength + len(blob))
+	return nil
+}
+
+// Size returns the current storage size of the memory cache in front of the
+// persistent database layer.
+func (db *Database) Size() common.StorageSize {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
+	return db.size
+}

trie/hasher.go

@@ -19,7 +19,6 @@ package trie
 import (
 	"bytes"
 	"hash"
-	"math/big"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -30,19 +29,21 @@ import (
 type hasher struct {
 	tmp        *bytes.Buffer
 	sha        hash.Hash
-	cachegen, cachelimit uint16
+	cachegen   uint16
+	cachelimit uint16
+	onleaf     LeafCallback
 }
 
-// hashers live in a global pool.
+// hashers live in a global db.
 var hasherPool = sync.Pool{
 	New: func() interface{} {
 		return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()}
 	},
 }
 
-func newHasher(cachegen, cachelimit uint16) *hasher {
+func newHasher(cachegen, cachelimit uint16, onleaf LeafCallback) *hasher {
 	h := hasherPool.Get().(*hasher)
-	h.cachegen, h.cachelimit = cachegen, cachelimit
+	h.cachegen, h.cachelimit, h.onleaf = cachegen, cachelimit, onleaf
 	return h
 }
 
@@ -52,10 +53,10 @@ func returnHasherToPool(h *hasher) {
 
 // hash collapses a node down into a hash node, also returning a copy of the
 // original node initialized with the computed hash to replace the original one.
-func (h *hasher) hash(n node, pool *NodePool, force bool) (node, node, error) {
+func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) {
 	// If we're not storing the node, just hashing, use available cached data
 	if hash, dirty := n.cache(); hash != nil {
-		if pool == nil {
+		if db == nil {
 			return hash, n, nil
 		}
 		if n.canUnload(h.cachegen, h.cachelimit) {
@@ -69,11 +70,11 @@ func (h *hasher) hash(n node, pool *NodePool, force bool) (node, node, error) {
 		}
 	}
 	// Trie not processed yet or needs storage, walk the children
-	collapsed, cached, refs, err := h.hashChildren(n, pool)
+	collapsed, cached, err := h.hashChildren(n, db)
 	if err != nil {
 		return hashNode{}, n, err
 	}
-	hashed, refs, err := h.store(collapsed, refs, pool, force)
+	hashed, err := h.store(collapsed, db, force)
 	if err != nil {
 		return hashNode{}, n, err
 	}
@@ -84,12 +85,12 @@ func (h *hasher) hash(n node, pool *NodePool, force bool) (node, node, error) {
 	switch cn := cached.(type) {
 	case *shortNode:
 		cn.flags.hash = cachedHash
-		if pool != nil {
+		if db != nil {
 			cn.flags.dirty = false
 		}
 	case *fullNode:
 		cn.flags.hash = cachedHash
-		if pool != nil {
+		if db != nil {
 			cn.flags.dirty = false
 		}
 	}
@@ -99,7 +100,7 @@ func (h *hasher) hash(n node, pool *NodePool, force bool) (node, node, error) {
 // hashChildren replaces the children of a node with their hashes if the encoded
 // size of the child is larger than a hash, returning the collapsed node as well
 // as a replacement for the original node with the child hashes cached in.
-func (h *hasher) hashChildren(original node, pool *NodePool) (node, node, []common.Hash, error) {
+func (h *hasher) hashChildren(original node, db *Database) (node, node, error) {
 	var err error
 
 	switch n := original.(type) {
@@ -110,15 +111,15 @@ func (h *hasher) hashChildren(original node, pool *NodePool) (node, node, []comm
 		cached.Key = common.CopyBytes(n.Key)
 
 		if _, ok := n.Val.(valueNode); !ok {
-			collapsed.Val, cached.Val, err = h.hash(n.Val, pool, false)
+			collapsed.Val, cached.Val, err = h.hash(n.Val, db, false)
 			if err != nil {
-				return original, original, nil, err
+				return original, original, err
 			}
 		}
 		if collapsed.Val == nil {
 			collapsed.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
 		}
-		return collapsed, cached, h.externals(collapsed.Val), nil
+		return collapsed, cached, nil
 
 	case *fullNode:
 		// Hash the full node's children, caching the newly hashed subtrees
@@ -126,9 +127,9 @@ func (h *hasher) hashChildren(original node, pool *NodePool) (node, node, []comm
 
 		for i := 0; i < 16; i++ {
 			if n.Children[i] != nil {
-				collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], pool, false)
+				collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false)
 				if err != nil {
-					return original, original, nil, err
+					return original, original, err
 				}
 			} else {
 				collapsed.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
@@ -138,65 +139,29 @@ func (h *hasher) hashChildren(original node, pool *NodePool) (node, node, []comm
 		if collapsed.Children[16] == nil {
 			collapsed.Children[16] = valueNode(nil)
 		}
-		var refs []common.Hash
+		return collapsed, cached, nil
-		for i := 0; i < 16; i++ {
-			refs = append(refs, h.externals(collapsed.Children[i])...)
-		}
-		return collapsed, cached, refs, nil
 
 	default:
 		// Value and hash nodes don't have children so they're left as were
-		return n, original, h.externals(n), nil
+		return n, original, nil
 	}
 }
 
-// externals returns any external nodes referenced by a particular node. The only
-// current case for it is when an account trie references its storage trie.
-func (h *hasher) externals(n node) []common.Hash {
-	// Only value nodes can reference external nodes
-	val, ok := n.(valueNode)
-	if !ok {
-		return nil
-	}
-	// Account nodes have very specific sizes, discard anything else
-	// TODO(karalabe): Seriously? Dafuq man?!
-	if size := len(val); size < 70 || size > 102 {
-		return nil
-	}
-	// Only account nodes can reference external storage tries
-	var account struct {
-		Nonce    uint64
-		Balance  *big.Int
-		Root     common.Hash
-		CodeHash []byte
-	}
-	if err := rlp.DecodeBytes(val, &account); err != nil {
-		//fmt.Printf(".")
-		return nil
-	}
-	// Empty tries are not referenced
-	if account.Root == emptyState {
-		return nil
-	}
-	return []common.Hash{account.Root}
-}
-
 // store hashes the node n and if we have a storage layer specified, it writes
 // the key/value pair to it and tracks any node->child references as well as any
 // node->external trie references.
-func (h *hasher) store(n node, refs []common.Hash, pool *NodePool, force bool) (node, []common.Hash, error) {
+func (h *hasher) store(n node, db *Database, force bool) (node, error) {
 	// Don't store hashes or empty nodes.
 	if _, isHash := n.(hashNode); n == nil || isHash {
-		return n, refs, nil
+		return n, nil
 	}
 	// Generate the RLP encoding of the node
 	h.tmp.Reset()
 	if err := rlp.Encode(h.tmp, n); err != nil {
 		panic("encode error: " + err.Error())
 	}
-
 	if h.tmp.Len() < 32 && !force {
-		return n, refs, nil // Nodes smaller than 32 bytes are stored inside their parent
+		return n, nil // Nodes smaller than 32 bytes are stored inside their parent
 	}
 	// Larger nodes are replaced by their hash and stored in the database.
 	hash, _ := n.cache()
@@ -205,31 +170,43 @@ func (h *hasher) store(n node, refs []common.Hash, pool *NodePool, force bool) (
 		h.sha.Write(h.tmp.Bytes())
 		hash = hashNode(h.sha.Sum(nil))
 	}
-	if pool != nil {
+	if db != nil {
 		// We are pooling the trie nodes into an intermediate memory cache
-		pool.lock.Lock()
+		db.lock.Lock()
-		defer pool.lock.Unlock()
 
 		hash := common.BytesToHash(hash)
-		pool.insert(hash, h.tmp.Bytes())
+		db.insert(hash, h.tmp.Bytes())
 
 		// Track all direct parent->child node references
 		switch n := n.(type) {
 		case *shortNode:
 			if child, ok := n.Val.(hashNode); ok {
-				pool.reference(common.BytesToHash(child), hash)
+				db.reference(common.BytesToHash(child), hash)
 			}
 		case *fullNode:
 			for i := 0; i < 16; i++ {
 				if child, ok := n.Children[i].(hashNode); ok {
-					pool.reference(common.BytesToHash(child), hash)
+					db.reference(common.BytesToHash(child), hash)
 				}
 			}
 		}
+		db.lock.Unlock()
+
 		// Track external references from account->storage trie
-		for _, ext := range refs {
+		if h.onleaf != nil {
-			pool.reference(ext, hash)
+			switch n := n.(type) {
+			case *shortNode:
+				if child, ok := n.Val.(valueNode); ok {
+					h.onleaf(child, hash)
+				}
+			case *fullNode:
+				for i := 0; i < 16; i++ {
+					if child, ok := n.Children[i].(valueNode); ok {
+						h.onleaf(child, hash)
 					}
 				}
-	return hash, nil, nil
+			}
+		}
+	}
+	return hash, nil
 }

trie/iterator_test.go

@@ -42,7 +42,7 @@ func TestIterator(t *testing.T) {
 		all[val.k] = val.v
 		trie.Update([]byte(val.k), []byte(val.v))
 	}
-	trie.Commit()
+	trie.Commit(nil)
 
 	found := make(map[string]string)
 	it := NewIterator(trie.NodeIterator(nil))
@@ -109,11 +109,16 @@ func TestNodeIteratorCoverage(t *testing.T) {
 	}
 	// Cross check the hashes and the database itself
 	for hash := range hashes {
-		if _, err := db.Get(hash.Bytes()); err != nil {
+		if _, err := db.Node(hash); err != nil {
 			t.Errorf("failed to retrieve reported node %x: %v", hash, err)
 		}
 	}
-	for _, key := range db.(*ethdb.MemDatabase).Keys() {
+	for _, hash := range db.Nodes() {
+		if _, ok := hashes[hash]; !ok {
+			t.Errorf("state entry not reported %x", hash)
+		}
+	}
+	for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() {
 		if _, ok := hashes[common.BytesToHash(key)]; !ok {
 			t.Errorf("state entry not reported %x", key)
 		}
@@ -191,13 +196,13 @@ func TestDifferenceIterator(t *testing.T) {
 	for _, val := range testdata1 {
 		triea.Update([]byte(val.k), []byte(val.v))
 	}
-	triea.Commit()
+	triea.Commit(nil)
 
 	trieb := newEmpty()
 	for _, val := range testdata2 {
 		trieb.Update([]byte(val.k), []byte(val.v))
 	}
-	trieb.Commit()
+	trieb.Commit(nil)
 
 	found := make(map[string]string)
 	di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil))
@@ -227,13 +232,13 @@ func TestUnionIterator(t *testing.T) {
 	for _, val := range testdata1 {
 		triea.Update([]byte(val.k), []byte(val.v))
 	}
-	triea.Commit()
+	triea.Commit(nil)
 
 	trieb := newEmpty()
 	for _, val := range testdata2 {
 		trieb.Update([]byte(val.k), []byte(val.v))
 	}
-	trieb.Commit()
+	trieb.Commit(nil)
 
 	di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
 	it := NewIterator(di)
@@ -278,35 +283,35 @@ func TestIteratorNoDups(t *testing.T) {
 }
 
 // This test checks that nodeIterator.Next can be retried after inserting missing trie nodes.
-func TestIteratorContinueAfterErrorDirect(t *testing.T) { testIteratorContinueAfterError(t, false) }
+func TestIteratorContinueAfterErrorDisk(t *testing.T)    { testIteratorContinueAfterError(t, false) }
-func TestIteratorContinueAfterErrorPooled(t *testing.T) { testIteratorContinueAfterError(t, true) }
+func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) }
 
-func testIteratorContinueAfterError(t *testing.T, pooled bool) {
+func testIteratorContinueAfterError(t *testing.T, memonly bool) {
-	var pool *NodePool
+	diskdb, _ := ethdb.NewMemDatabase()
-	if pooled {
+	triedb := NewDatabase(diskdb)
-		pool = NewNodePool()
-	}
-	db, _ := ethdb.NewMemDatabase()
 
-	tr, _ := New(common.Hash{}, db, pool)
+	tr, _ := New(common.Hash{}, triedb)
 	for _, val := range testdata1 {
 		tr.Update([]byte(val.k), []byte(val.v))
 	}
-	tr.Commit()
+	tr.Commit(nil)
+	if !memonly {
+		triedb.Commit(tr.Hash(), diskdb)
+	}
 	wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
 
 	var (
-		dbKeys   [][]byte
+		diskKeys [][]byte
-		poolKeys []common.Hash
+		memKeys  []common.Hash
 	)
-	if pooled {
+	if memonly {
-		poolKeys = pool.Nodes()
+		memKeys = triedb.Nodes()
 	} else {
-		dbKeys = db.Keys()
+		diskKeys = diskdb.Keys()
 	}
 	for i := 0; i < 20; i++ {
 		// Create trie that will load all nodes from DB.
-		tr, _ := New(tr.Hash(), db, pool)
+		tr, _ := New(tr.Hash(), triedb)
 
 		// Remove a random node from the database. It can't be the root node
 		// because that one is already loaded.
@@ -315,21 +320,21 @@ func testIteratorContinueAfterError(t *testing.T, pooled bool) {
 			rval []byte
 		)
 		for {
-			if pooled {
+			if memonly {
-				rkey = poolKeys[rand.Intn(len(poolKeys))]
+				rkey = memKeys[rand.Intn(len(memKeys))]
 			} else {
-				copy(rkey[:], dbKeys[rand.Intn(len(dbKeys))])
+				copy(rkey[:], diskKeys[rand.Intn(len(diskKeys))])
 			}
 			if rkey != tr.Hash() {
 				break
 			}
 		}
-		if pooled {
+		if memonly {
-			rval, _ = pool.cache[rkey]
+			rval, _ = triedb.nodes[rkey]
-			delete(pool.cache, rkey)
+			delete(triedb.nodes, rkey)
 		} else {
-			rval, _ = db.Get(rkey[:])
+			rval, _ = diskdb.Get(rkey[:])
-			db.Delete(rkey[:])
+			diskdb.Delete(rkey[:])
 		}
 		// Iterate until the error is hit.
 		seen := make(map[string]bool)
@@ -341,10 +346,10 @@ func testIteratorContinueAfterError(t *testing.T, pooled bool) {
 		}
 
 		// Add the node back and continue iteration.
-		if pooled {
+		if memonly {
-			pool.cache[rkey] = rval
+			triedb.nodes[rkey] = rval
 		} else {
-			db.Put(rkey[:], rval)
+			diskdb.Put(rkey[:], rval)
 		}
 		checkIteratorNoDups(t, it, seen)
 		if it.Error() != nil {
@@ -359,40 +364,39 @@ func testIteratorContinueAfterError(t *testing.T, pooled bool) {
 // Similar to the test above, this one checks that failure to create nodeIterator at a
 // certain key prefix behaves correctly when Next is called. The expectation is that Next
 // should retry seeking before returning true for the first time.
-func TestIteratorContinueAfterSeekErrorDirect(t *testing.T) {
+func TestIteratorContinueAfterSeekErrorDisk(t *testing.T) {
 	testIteratorContinueAfterSeekError(t, false)
 }
-func TestIteratorContinueAfterSeekErrorPooled(t *testing.T) {
+func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) {
 	testIteratorContinueAfterSeekError(t, true)
 }
 
-func testIteratorContinueAfterSeekError(t *testing.T, pooled bool) {
+func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
 	// Commit test trie to db, then remove the node containing "bars".
-	var pool *NodePool
+	diskdb, _ := ethdb.NewMemDatabase()
-	if pooled {
+	triedb := NewDatabase(diskdb)
-		pool = NewNodePool()
-	}
-	db, _ := ethdb.NewMemDatabase()
 
-	ctr, _ := New(common.Hash{}, db, pool)
+	ctr, _ := New(common.Hash{}, triedb)
 	for _, val := range testdata1 {
 		ctr.Update([]byte(val.k), []byte(val.v))
 	}
-	root, _ := ctr.Commit()
+	root, _ := ctr.Commit(nil)
-
+	if !memonly {
+		triedb.Commit(root, diskdb)
+	}
 	barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
 	var barNodeBlob []byte
 
-	if pooled {
+	if memonly {
-		barNodeBlob = pool.cache[barNodeHash]
+		barNodeBlob = triedb.nodes[barNodeHash]
-		delete(pool.cache, barNodeHash)
+		delete(triedb.nodes, barNodeHash)
 	} else {
-		barNodeBlob, _ = db.Get(barNodeHash[:])
+		barNodeBlob, _ = diskdb.Get(barNodeHash[:])
-		db.Delete(barNodeHash[:])
+		diskdb.Delete(barNodeHash[:])
 	}
 	// Create a new iterator that seeks to "bars". Seeking can't proceed because
 	// the node is missing.
-	tr, _ := New(root, db, pool)
+	tr, _ := New(root, triedb)
 	it := tr.NodeIterator([]byte("bars"))
 	missing, ok := it.Error().(*MissingNodeError)
 	if !ok {
@@ -401,10 +405,10 @@ func testIteratorContinueAfterSeekError(t *testing.T, pooled bool) {
 		t.Fatal("wrong node missing")
 	}
 	// Reinsert the missing node.
-	if pooled {
+	if memonly {
-		pool.cache[barNodeHash] = barNodeBlob
+		triedb.nodes[barNodeHash] = barNodeBlob
 	} else {
-		db.Put(barNodeHash[:], barNodeBlob)
+		diskdb.Put(barNodeHash[:], barNodeBlob)
 	}
 	// Check that iteration produces the right set of values.
 	if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {

trie/node_pool.go

@@ -1,195 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package trie
-
-import (
-	"sync"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-// NodePool is an intermediate write layer between the trie data structures and
-// the disk database. The aim is to accumulate trie writes in-memory and only
-// periodically flush a couple tries to disk, garbage collecting the remainder.
-type NodePool struct {
-	cache map[common.Hash][]byte // Cached data blocks of the trie nodes
-
-	parents  map[common.Hash]int                      // Number of live nodes referencing a given one
-	children map[common.Hash]map[common.Hash]struct{} // Set of children referenced by given nodes
-
-	gctime  time.Duration      // Time spent on garbage collection since last commit
-	gcnodes uint64             // Nodes garbage collected since last commit
-	gcsize  common.StorageSize // Data storage garbage collected since last commit
-
-	size common.StorageSize // Storage size of the memory pool
-	lock sync.RWMutex
-}
-
-// NewNodePool creates a new memory pool to store ephemeral trie nodes before they
-// are written out to disk or garbage collected.
-func NewNodePool() *NodePool {
-	pool := &NodePool{
-		cache:    make(map[common.Hash][]byte),
-		parents:  make(map[common.Hash]int),
-		children: make(map[common.Hash]map[common.Hash]struct{}),
-	}
-	pool.children[common.Hash{}] = make(map[common.Hash]struct{})
-	return pool
-}
-
-// insert writes a new trie node to the memory pool if it's yet unknown. The pool
-// will make a copy of the slice.
-//
-// Note, this method assumes that the pool's lock is held!
-func (pool *NodePool) insert(hash common.Hash, blob []byte) {
-	if _, ok := pool.cache[hash]; ok {
-		return
-	}
-	pool.cache[hash] = common.CopyBytes(blob)
-	pool.children[hash] = make(map[common.Hash]struct{})
-
-	pool.size += common.StorageSize(common.HashLength + len(blob))
-}
-
-// Fetch retrieves a cached trie node from memory, or returns nil if the pool
-// does not have this particular piece of data.
-func (pool *NodePool) Fetch(hash common.Hash) []byte {
-	pool.lock.RLock()
-	defer pool.lock.RUnlock()
-
-	return pool.cache[hash]
-}
-
-// Nodes retrieves the hashes of all the nodes cached within the node pool. This
-// method is extremely expensive and should only be used in test code to validate
-// internal states.
-func (pool *NodePool) Nodes() []common.Hash {
-	pool.lock.RLock()
-	defer pool.lock.RUnlock()
-
-	var hashes = make([]common.Hash, 0, len(pool.cache))
-	for hash := range pool.cache {
-		hashes = append(hashes, hash)
-	}
-	return hashes
-}
-
-// Reference adds a new reference from parent to node.
-func (pool *NodePool) Reference(node common.Hash, parent common.Hash) {
-	pool.lock.RLock()
-	defer pool.lock.RUnlock()
-
-	pool.reference(node, parent)
-}
-
-// reference is the private locked version of Reference.
-func (pool *NodePool) reference(node common.Hash, parent common.Hash) {
-	// If the node does not exist, it's a node pulled from disk, skip
-	if _, ok := pool.cache[node]; !ok {
-		return
-	}
-	pool.parents[node]++
-	pool.children[parent][node] = struct{}{}
-}
-
-// Dereference removes an existing reference from parent to node.
-func (pool *NodePool) Dereference(node common.Hash, parent common.Hash) {
-	pool.lock.Lock()
-	defer pool.lock.Unlock()
-
-	nodes, storage, start := len(pool.cache), pool.size, time.Now()
-	pool.dereference(node, parent)
-
-	pool.gcnodes += uint64(nodes - len(pool.cache))
-	pool.gcsize += storage - pool.size
-	pool.gctime += time.Since(start)
-}
-
-// dereference is the private locked version of Dereference.
-func (pool *NodePool) dereference(node common.Hash, parent common.Hash) {
-	// If the node does not exist, it's a previously comitted node.
-	blob, ok := pool.cache[node]
-	if !ok {
-		return
-	}
-	delete(pool.children[parent], node)
-	pool.parents[node]--
-
-	// If there are no more references to the child, delete it and cascade
-	if pool.parents[node] == 0 {
-		for child := range pool.children[node] {
-			pool.dereference(child, node)
-		}
-		delete(pool.cache, node)
-		delete(pool.parents, node)
-		delete(pool.children, node)
-
-		pool.size -= common.StorageSize(common.HashLength + len(blob))
-	}
-}
-
-// Commit iterates over all the children of a particular node, writes them out
-// to disk, forcefully tearing down all references in both directions.
-func (pool *NodePool) Commit(node common.Hash, db DatabaseWriter) error {
-	pool.lock.Lock()
-	defer pool.lock.Unlock()
-
-	nodes, storage, start := len(pool.cache), pool.size, time.Now()
-	if err := pool.commit(node, db); err != nil {
-		log.Error("Failed to commit trie from mempool", "err", err)
-		return err
-	}
-	log.Debug("Committed trie from memory pool", "nodes", nodes-len(pool.cache), "size", storage-pool.size, "time", time.Since(start),
-		"gcnodes", pool.gcnodes, "gcsize", pool.gcsize, "gctime", pool.gctime, "livenodes", len(pool.cache), "livesize", pool.size)
-
-	// Reset the garbage collection statistics
-	pool.gcnodes, pool.gcsize, pool.gctime = 0, 0, 0
-
-	// Sanity check that we don't have dangling nodes in the pool (missing refs)
-	for hash, refs := range pool.parents {
-		if refs == 0 {
-			log.Warn("dangling node in mempool", "hash", hash)
-			break
-		}
-	}
-	return nil
-}
-
-// commit is the private locked version of Commit.
-func (pool *NodePool) commit(node common.Hash, db DatabaseWriter) error {
-	// If the node does not exist, it's a previously comitted node.
-	blob, ok := pool.cache[node]
-	if !ok {
-		return nil
-	}
-	for child := range pool.children[node] {
-		if err := pool.commit(child, db); err != nil {
-			return err
-		}
-	}
-	if err := db.Put(node[:], blob); err != nil {
-		return err
-	}
-	delete(pool.cache, node)
-	delete(pool.parents, node)
-	delete(pool.children, node)
-
-	pool.size -= common.StorageSize(common.HashLength + len(blob))
-	return nil
-}

trie/proof.go

@@ -26,15 +26,13 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-// Prove constructs a merkle proof for key. The result contains all
+// Prove constructs a merkle proof for key. The result contains all encoded nodes
-// encoded nodes on the path to the value at key. The value itself is
+// on the path to the value at key. The value itself is also included in the last
-// also included in the last node and can be retrieved by verifying
+// node and can be retrieved by verifying the proof.
-// the proof.
 //
-// If the trie does not contain a value for key, the returned proof
+// If the trie does not contain a value for key, the returned proof contains all
-// contains all nodes of the longest existing prefix of the key
+// nodes of the longest existing prefix of the key (at least the root node), ending
-// (at least the root node), ending with the node that proves the
+// with the node that proves the absence of the key.
-// absence of the key.
 func (t *Trie) Prove(key []byte, fromLevel uint, proofDb DatabaseWriter) error {
 	// Collect all nodes on the path to key.
 	key = keybytesToHex(key)
@@ -66,12 +64,12 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb DatabaseWriter) error {
 			panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
 		}
 	}
-	hasher := newHasher(0, 0)
+	hasher := newHasher(0, 0, nil)
 	for i, n := range nodes {
 		// Don't bother checking for errors here since hasher panics
 		// if encoding doesn't work and we're not writing to any database.
-		n, _, _, _ = hasher.hashChildren(n, nil)
+		n, _, _ = hasher.hashChildren(n, nil)
-		hn, _, _ := hasher.store(n, nil, nil, false)
+		hn, _ := hasher.store(n, nil, false)
 		if hash, ok := hn.(hashNode); ok || i == 0 {
 			// If the node's database encoding is a hash (or is the
 			// root node), it becomes a proof element.
@@ -89,19 +87,18 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb DatabaseWriter) error {
 	return nil
 }
 
-// VerifyProof checks merkle proofs. The given proof must contain the
+// VerifyProof checks merkle proofs. The given proof must contain the value for
-// value for key in a trie with the given root hash. VerifyProof
+// key in a trie with the given root hash. VerifyProof returns an error if the
-// returns an error if the proof contains invalid trie nodes or the
+// proof contains invalid trie nodes or the wrong value.
-// wrong value.
 func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, err error, nodes int) {
 	key = keybytesToHex(key)
-	wantHash := rootHash[:]
+	wantHash := rootHash
 	for i := 0; ; i++ {
-		buf, _ := proofDb.Get(wantHash)
+		buf, _ := proofDb.Get(wantHash[:])
 		if buf == nil {
-			return nil, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash[:]), i
+			return nil, fmt.Errorf("proof node %d (hash %064x) missing", i, wantHash), i
 		}
-		n, err := decodeNode(wantHash, buf, 0)
+		n, err := decodeNode(wantHash[:], buf, 0)
 		if err != nil {
 			return nil, fmt.Errorf("bad proof node %d: %v", i, err), i
 		}
@@ -112,7 +109,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (valu
 			return nil, nil, i
 		case hashNode:
 			key = keyrest
-			wantHash = cld
+			copy(wantHash[:], cld)
 		case valueNode:
 			return cld, nil, i + 1
 		}

trie/secure_trie.go

@@ -23,10 +23,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
-var secureKeyPrefix = []byte("secure-key-")
-
-const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
-
 // SecureTrie wraps a trie with key hashing. In a secure trie, all
 // access operations hash the key using keccak256. This prevents
 // calling code from creating long chains of nodes that
@@ -39,8 +35,7 @@ const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
 // SecureTrie is not safe for concurrent use.
 type SecureTrie struct {
 	trie             Trie
-	hashKeyBuf       [secureKeyLength]byte
+	hashKeyBuf       [common.HashLength]byte
-	secKeyBuf        [200]byte
 	secKeyCache      map[string][]byte
 	secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
 }
@@ -56,11 +51,11 @@ type SecureTrie struct {
 // Loaded nodes are kept around until their 'cache generation' expires.
 // A new cache generation is created by each call to Commit.
 // cachelimit sets the number of past cache generations to keep.
-func NewSecure(root common.Hash, db Database, pool *NodePool, cachelimit uint16) (*SecureTrie, error) {
+func NewSecure(root common.Hash, db *Database, cachelimit uint16) (*SecureTrie, error) {
 	if db == nil {
-		panic("NewSecure called with nil database")
+		panic("trie.NewSecure called without a database")
 	}
-	trie, err := New(root, db, pool)
+	trie, err := New(root, db)
 	if err != nil {
 		return nil, err
 	}
@@ -136,7 +131,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
 	if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
 		return key
 	}
-	key, _ := t.trie.db.Get(t.secKey(shaKey))
+	key, _ := t.trie.db.preimage(common.BytesToHash(shaKey))
 	return key
 }
 
@@ -145,8 +140,19 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
 //
 // Committing flushes nodes from memory. Subsequent Get calls will load nodes
 // from the database.
-func (t *SecureTrie) Commit() (root common.Hash, err error) {
+func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
-	return t.CommitTo(t.trie.db)
+	// Write all the pre-images to the actual disk database
+	if len(t.getSecKeyCache()) > 0 {
+		t.trie.db.lock.Lock()
+		for hk, key := range t.secKeyCache {
+			t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key)
+		}
+		t.trie.db.lock.Unlock()
+
+		t.secKeyCache = make(map[string][]byte)
+	}
+	// Commit the trie to its intermediate node database
+	return t.trie.Commit(onleaf)
 }
 
 func (t *SecureTrie) Hash() common.Hash {
@@ -168,38 +174,11 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator {
 	return t.trie.NodeIterator(start)
 }
 
-// CommitTo writes all nodes and the secure hash pre-images to the given database.
-// Nodes are stored with their sha3 hash as the key.
-//
-// Committing flushes nodes from memory. Subsequent Get calls will load nodes from
-// the trie's database. Calling code must ensure that the changes made to db are
-// written back to the trie's attached database before using the trie.
-func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
-	if len(t.getSecKeyCache()) > 0 {
-		for hk, key := range t.secKeyCache {
-			if err := db.Put(t.secKey([]byte(hk)), key); err != nil {
-				return common.Hash{}, err
-			}
-		}
-		t.secKeyCache = make(map[string][]byte)
-	}
-	return t.trie.CommitTo(db)
-}
-
-// secKey returns the database key for the preimage of key, as an ephemeral buffer.
-// The caller must not hold onto the return value because it will become
-// invalid on the next call to hashKey or secKey.
-func (t *SecureTrie) secKey(key []byte) []byte {
-	buf := append(t.secKeyBuf[:0], secureKeyPrefix...)
-	buf = append(buf, key...)
-	return buf
-}
-
 // hashKey returns the hash of key as an ephemeral buffer.
 // The caller must not hold onto the return value because it will become
 // invalid on the next call to hashKey or secKey.
 func (t *SecureTrie) hashKey(key []byte) []byte {
-	h := newHasher(0, 0)
+	h := newHasher(0, 0, nil)
 	h.sha.Reset()
 	h.sha.Write(key)
 	buf := h.sha.Sum(t.hashKeyBuf[:0])

trie/secure_trie_test.go

@@ -28,16 +28,20 @@ import (
 )
 
 func newEmptySecure() *SecureTrie {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	trie, _ := NewSecure(common.Hash{}, db, NewNodePool(), 0)
+	triedb := NewDatabase(diskdb)
+
+	trie, _ := NewSecure(common.Hash{}, triedb, 0)
 	return trie
 }
 
 // makeTestSecureTrie creates a large enough secure trie for testing.
-func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
+func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
 	// Create an empty trie
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	trie, _ := NewSecure(common.Hash{}, db, NewNodePool(), 0)
+	triedb := NewDatabase(diskdb)
+
+	trie, _ := NewSecure(common.Hash{}, triedb, 0)
 
 	// Fill it with some arbitrary data
 	content := make(map[string][]byte)
@@ -58,10 +62,10 @@ func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
 			trie.Update(key, val)
 		}
 	}
-	trie.Commit()
+	trie.Commit(nil)
 
 	// Return the generated trie
-	return db, trie, content
+	return triedb, trie, content
 }
 
 func TestSecureDelete(t *testing.T) {
@@ -137,7 +141,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
 					tries[index].Update(key, val)
 				}
 			}
-			tries[index].Commit()
+			tries[index].Commit(nil)
 		}(i)
 	}
 	// Wait for all threads to finish

trie/sync.go

@@ -42,7 +42,7 @@ type request struct {
 	depth   int        // Depth level within the trie the node is located to prioritise DFS
 	deps    int        // Number of dependencies before allowed to commit this node
 
-	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
+	callback LeafCallback // Callback to invoke if a leaf node it reached on this branch
 }
 
 // SyncResult is a simple list to return missing nodes along with their request
@@ -67,11 +67,6 @@ func newSyncMemBatch() *syncMemBatch {
 	}
 }
 
-// TrieSyncLeafCallback is a callback type invoked when a trie sync reaches a
-// leaf node. It's used by state syncing to check if the leaf node requires some
-// further data syncing.
-type TrieSyncLeafCallback func(leaf []byte, parent common.Hash) error
-
 // TrieSync is the main state trie synchronisation scheduler, which provides yet
 // unknown trie hashes to retrieve, accepts node data associated with said hashes
 // and reconstructs the trie step by step until all is done.
@@ -83,7 +78,7 @@ type TrieSync struct {
 }
 
 // NewTrieSync creates a new trie data download scheduler.
-func NewTrieSync(root common.Hash, database DatabaseReader, callback TrieSyncLeafCallback) *TrieSync {
+func NewTrieSync(root common.Hash, database DatabaseReader, callback LeafCallback) *TrieSync {
 	ts := &TrieSync{
 		database: database,
 		membatch: newSyncMemBatch(),
@@ -95,7 +90,7 @@ func NewTrieSync(root common.Hash, database DatabaseReader, callback TrieSyncLea
 }
 
 // AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
-func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
+func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback LeafCallback) {
 	// Short circuit if the trie is empty or already known
 	if root == emptyRoot {
 		return

trie/sync_test.go

@@ -25,10 +25,11 @@ import (
 )
 
 // makeTestTrie create a sample test trie to test node-wise reconstruction.
-func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
+func makeTestTrie() (*Database, *Trie, map[string][]byte) {
 	// Create an empty trie
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	trie, _ := New(common.Hash{}, db, nil)
+	triedb := NewDatabase(diskdb)
+	trie, _ := New(common.Hash{}, triedb)
 
 	// Fill it with some arbitrary data
 	content := make(map[string][]byte)
@@ -49,17 +50,17 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
 			trie.Update(key, val)
 		}
 	}
-	trie.Commit()
+	trie.Commit(nil)
 
 	// Return the generated trie
-	return db, trie, content
+	return triedb, trie, content
 }
 
 // checkTrieContents cross references a reconstructed trie with an expected data
 // content map.
-func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
+func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) {
 	// Check root availability and trie contents
-	trie, err := New(common.BytesToHash(root), db, nil)
+	trie, err := New(common.BytesToHash(root), db)
 	if err != nil {
 		t.Fatalf("failed to create trie at %x: %v", root, err)
 	}
@@ -74,9 +75,9 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
 }
 
 // checkTrieConsistency checks that all nodes in a trie are indeed present.
-func checkTrieConsistency(db Database, root common.Hash) error {
+func checkTrieConsistency(db *Database, root common.Hash) error {
 	// Create and iterate a trie rooted in a subnode
-	trie, err := New(root, db, nil)
+	trie, err := New(root, db)
 	if err != nil {
 		return nil // Consider a non existent state consistent
 	}
@@ -88,12 +89,18 @@ func checkTrieConsistency(db Database, root common.Hash) error {
 
 // Tests that an empty trie is not scheduled for syncing.
 func TestEmptyTrieSync(t *testing.T) {
-	emptyA, _ := New(common.Hash{}, nil, nil)
+	diskdbA, _ := ethdb.NewMemDatabase()
-	emptyB, _ := New(emptyRoot, nil, nil)
+	triedbA := NewDatabase(diskdbA)
+
+	diskdbB, _ := ethdb.NewMemDatabase()
+	triedbB := NewDatabase(diskdbB)
+
+	emptyA, _ := New(common.Hash{}, triedbA)
+	emptyB, _ := New(emptyRoot, triedbB)
 
 	for i, trie := range []*Trie{emptyA, emptyB} {
-		db, _ := ethdb.NewMemDatabase()
+		diskdb, _ := ethdb.NewMemDatabase()
-		if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
+		if req := NewTrieSync(trie.Hash(), diskdb, nil).Missing(1); len(req) != 0 {
 			t.Errorf("test %d: content requested for empty trie: %v", i, req)
 		}
 	}
@@ -109,14 +116,15 @@ func testIterativeTrieSync(t *testing.T, batch int) {
 	srcDb, srcTrie, srcData := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(batch)...)
 	for len(queue) > 0 {
 		results := make([]SyncResult, len(queue))
 		for i, hash := range queue {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -125,13 +133,13 @@ func testIterativeTrieSync(t *testing.T, batch int) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		queue = append(queue[:0], sched.Missing(batch)...)
 	}
 	// Cross check that the two tries are in sync
-	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+	checkTrieContents(t, triedb, srcTrie.Root(), srcData)
 }
 
 // Tests that the trie scheduler can correctly reconstruct the state even if only
@@ -141,15 +149,16 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
 	srcDb, srcTrie, srcData := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(10000)...)
 	for len(queue) > 0 {
 		// Sync only half of the scheduled nodes
 		results := make([]SyncResult, len(queue)/2+1)
 		for i, hash := range queue[:len(results)] {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -158,13 +167,13 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		queue = append(queue[len(results):], sched.Missing(10000)...)
 	}
 	// Cross check that the two tries are in sync
-	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+	checkTrieContents(t, triedb, srcTrie.Root(), srcData)
 }
 
 // Tests that given a root hash, a trie can sync iteratively on a single thread,
@@ -178,8 +187,9 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 	srcDb, srcTrie, srcData := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(batch) {
@@ -189,7 +199,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 		// Fetch all the queued nodes in a random order
 		results := make([]SyncResult, 0, len(queue))
 		for hash := range queue {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -199,7 +209,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		queue = make(map[common.Hash]struct{})
@@ -208,7 +218,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 		}
 	}
 	// Cross check that the two tries are in sync
-	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+	checkTrieContents(t, triedb, srcTrie.Root(), srcData)
 }
 
 // Tests that the trie scheduler can correctly reconstruct the state even if only
@@ -218,8 +228,9 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 	srcDb, srcTrie, srcData := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	queue := make(map[common.Hash]struct{})
 	for _, hash := range sched.Missing(10000) {
@@ -229,7 +240,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 		// Sync only half of the scheduled nodes, even those in random order
 		results := make([]SyncResult, 0, len(queue)/2+1)
 		for hash := range queue {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -243,7 +254,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		for _, result := range results {
@@ -254,7 +265,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 		}
 	}
 	// Cross check that the two tries are in sync
-	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+	checkTrieContents(t, triedb, srcTrie.Root(), srcData)
 }
 
 // Tests that a trie sync will not request nodes multiple times, even if they
@@ -264,8 +275,9 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 	srcDb, srcTrie, srcData := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	queue := append([]common.Hash{}, sched.Missing(0)...)
 	requested := make(map[common.Hash]struct{})
@@ -273,7 +285,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 	for len(queue) > 0 {
 		results := make([]SyncResult, len(queue))
 		for i, hash := range queue {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -287,13 +299,13 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		queue = append(queue[:0], sched.Missing(0)...)
 	}
 	// Cross check that the two tries are in sync
-	checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+	checkTrieContents(t, triedb, srcTrie.Root(), srcData)
 }
 
 // Tests that at any point in time during a sync, only complete sub-tries are in
@@ -303,8 +315,9 @@ func TestIncompleteTrieSync(t *testing.T) {
 	srcDb, srcTrie, _ := makeTestTrie()
 
 	// Create a destination trie and sync with the scheduler
-	dstDb, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+	triedb := NewDatabase(diskdb)
+	sched := NewTrieSync(srcTrie.Hash(), diskdb, nil)
 
 	added := []common.Hash{}
 	queue := append([]common.Hash{}, sched.Missing(1)...)
@@ -312,7 +325,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 		// Fetch a batch of trie nodes
 		results := make([]SyncResult, len(queue))
 		for i, hash := range queue {
-			data, err := srcDb.Get(hash.Bytes())
+			data, err := srcDb.Node(hash)
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
@@ -322,7 +335,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
 		}
-		if index, err := sched.Commit(dstDb); err != nil {
+		if index, err := sched.Commit(diskdb); err != nil {
 			t.Fatalf("failed to commit data #%d: %v", index, err)
 		}
 		for _, result := range results {
@@ -330,7 +343,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 		}
 		// Check that all known sub-tries in the synced trie are complete
 		for _, root := range added {
-			if err := checkTrieConsistency(dstDb, root); err != nil {
+			if err := checkTrieConsistency(triedb, root); err != nil {
 				t.Fatalf("trie inconsistent: %v", err)
 			}
 		}
@@ -340,12 +353,12 @@ func TestIncompleteTrieSync(t *testing.T) {
 	// Sanity check that removing any node from the database is detected
 	for _, node := range added[1:] {
 		key := node.Bytes()
-		value, _ := dstDb.Get(key)
+		value, _ := diskdb.Get(key)
 
-		dstDb.Delete(key)
+		diskdb.Delete(key)
-		if err := checkTrieConsistency(dstDb, added[0]); err == nil {
+		if err := checkTrieConsistency(triedb, added[0]); err == nil {
 			t.Fatalf("trie inconsistency not caught, missing: %x", key)
 		}
-		dstDb.Put(key, value)
+		diskdb.Put(key, value)
 	}
 }

trie/trie.go

@@ -22,16 +22,17 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/rcrowley/go-metrics"
 )
 
 var (
-	// This is the known root hash of an empty trie.
+	// emptyRoot is the known root hash of an empty trie.
 	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
-	// This is the known hash of an empty state trie entry.
+
-	emptyState common.Hash
+	// emptyState is the known hash of an empty state trie entry.
+	emptyState = crypto.Keccak256Hash(nil)
 )
 
 var (
@@ -53,29 +54,10 @@ func CacheUnloads() int64 {
 	return cacheUnloadCounter.Count()
 }
 
-func init() {
+// LeafCallback is a callback type invoked when a trie operation reaches a leaf
-	sha3.NewKeccak256().Sum(emptyState[:0])
+// node. It's used by state sync and commit to allow handling external references
-}
+// between account and storage tries.
-
+type LeafCallback func(leaf []byte, parent common.Hash) error
-// Database must be implemented by backing stores for the trie.
-type Database interface {
-	DatabaseReader
-	DatabaseWriter
-}
-
-// DatabaseReader wraps the Get method of a backing store for the trie.
-type DatabaseReader interface {
-	Get(key []byte) (value []byte, err error)
-	Has(key []byte) (bool, error)
-}
-
-// DatabaseWriter wraps the Put method of a backing store for the trie.
-type DatabaseWriter interface {
-	// Put stores the mapping key->value in the database.
-	// Implementations must not hold onto the value bytes, the trie
-	// will reuse the slice across calls to Put.
-	Put(key, value []byte) error
-}
 
 // Trie is a Merkle Patricia Trie.
 // The zero value is an empty trie with no database.
@@ -83,9 +65,8 @@ type DatabaseWriter interface {
 //
 // Trie is not safe for concurrent use.
 type Trie struct {
+	db           *Database
 	root         node
-	db           Database
-	pool         *NodePool
 	originalRoot common.Hash
 
 	// Cache generation values.
@@ -112,12 +93,15 @@ func (t *Trie) newFlag() nodeFlag {
 // trie is initially empty and does not require a database. Otherwise,
 // New will panic if db is nil and returns a MissingNodeError if root does
 // not exist in the database. Accessing the trie loads nodes from db on demand.
-func New(root common.Hash, db Database, pool *NodePool) (*Trie, error) {
+func New(root common.Hash, db *Database) (*Trie, error) {
-	trie := &Trie{db: db, pool: pool, originalRoot: root}
-	if (root != common.Hash{}) && root != emptyRoot {
 	if db == nil {
-			panic("trie.New: cannot use existing root without a database")
+		panic("trie.New called without a database")
 	}
+	trie := &Trie{
+		db:           db,
+		originalRoot: root,
+	}
+	if (root != common.Hash{}) && root != emptyRoot {
 		rootnode, err := trie.resolveHash(root[:], nil)
 		if err != nil {
 			return nil, err
@@ -448,15 +432,9 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
 func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
 	cacheMissCounter.Inc(1)
 
-	// Try to load the node from the recent mempool
 	hash := common.BytesToHash(n)
-	if t.pool != nil {
+
-		if enc := t.pool.Fetch(hash); enc != nil {
+	enc, err := t.db.Node(hash)
-			return mustDecodeNode(n, enc, t.cachegen), nil
-		}
-	}
-	// Node not in the mempool, load it from disk
-	enc, err := t.db.Get(n)
 	if err != nil || enc == nil {
 		return nil, &MissingNodeError{NodeHash: hash, Path: prefix}
 	}
@@ -470,47 +448,18 @@ func (t *Trie) Root() []byte { return t.Hash().Bytes() }
 // Hash returns the root hash of the trie. It does not write to the
 // database and can be used even if the trie doesn't have one.
 func (t *Trie) Hash() common.Hash {
-	hash, cached, _ := t.hashRoot(nil)
+	hash, cached, _ := t.hashRoot(nil, nil)
 	t.root = cached
 	return common.BytesToHash(hash.(hashNode))
 }
 
-// Commit writes all nodes to the trie's database.
+// Commit writes all nodes to the trie's memory database, tracking the internal
-// Nodes are stored with their sha3 hash as the key.
+// and external (for account tries) references.
-//
+func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
-// Committing flushes nodes from memory.
-// Subsequent Get calls will load nodes from the database.
-func (t *Trie) Commit() (root common.Hash, err error) {
 	if t.db == nil {
-		panic("Commit called on trie with nil database")
+		panic("commit called on trie with nil database")
 	}
-	return t.CommitTo(t.db)
+	hash, cached, err := t.hashRoot(t.db, onleaf)
-}
-
-// CommitTo writes all nodes to the given database.
-// Nodes are stored with their sha3 hash as the key.
-//
-// Committing flushes nodes from memory. Subsequent Get calls will
-// load nodes from the trie's database. Calling code must ensure that
-// the changes made to db are written back to the trie's attached
-// database before using the trie.
-func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
-	// Retrieve the intermedia trie node memory cache if really writing
-	var pool *NodePool
-	if db != nil {
-		if pool = t.pool; pool == nil {
-			// If the trie has no intermediate memory pool, but actual database write was
-			// nonetheless requested, store into an emphemeral pool and flush out to disk.
-			pool = NewNodePool()
-			defer func() {
-				for hash, blob := range pool.cache {
-					db.Put(hash[:], blob)
-				}
-			}()
-		}
-	}
-	// Calculate the root hash and store in the mempool if requested
-	hash, cached, err := t.hashRoot(pool)
 	if err != nil {
 		return common.Hash{}, err
 	}
@@ -519,11 +468,11 @@ func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
 	return common.BytesToHash(hash.(hashNode)), nil
 }
 
-func (t *Trie) hashRoot(pool *NodePool) (node, node, error) {
+func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) {
 	if t.root == nil {
 		return hashNode(emptyRoot.Bytes()), nil, nil
 	}
-	h := newHasher(t.cachegen, t.cachelimit)
+	h := newHasher(t.cachegen, t.cachelimit, onleaf)
 	defer returnHasherToPool(h)
-	return h.hash(t.root, pool, true)
+	return h.hash(t.root, db, true)
 }

trie/trie_test.go

@@ -43,8 +43,8 @@ func init() {
 
 // Used for testing
 func newEmpty() *Trie {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	trie, _ := New(common.Hash{}, db, NewNodePool())
+	trie, _ := New(common.Hash{}, NewDatabase(diskdb))
 	return trie
 }
 
@@ -68,8 +68,8 @@ func TestNull(t *testing.T) {
 }
 
 func TestMissingRoot(t *testing.T) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db, NewNodePool())
+	trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(diskdb))
 	if trie != nil {
 		t.Error("New returned non-nil trie for invalid root")
 	}
@@ -78,75 +78,75 @@ func TestMissingRoot(t *testing.T) {
 	}
 }
 
-func TestMissingNodeDirect(t *testing.T) { testMissingNode(t, false) }
+func TestMissingNodeDisk(t *testing.T)    { testMissingNode(t, false) }
-func TestMissingNodePooled(t *testing.T) { testMissingNode(t, true) }
+func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) }
 
-func testMissingNode(t *testing.T, pooled bool) {
+func testMissingNode(t *testing.T, memonly bool) {
-	var pool *NodePool
+	diskdb, _ := ethdb.NewMemDatabase()
-	if pooled {
+	triedb := NewDatabase(diskdb)
-		pool = NewNodePool()
-	}
-	db, _ := ethdb.NewMemDatabase()
 
-	trie, _ := New(common.Hash{}, db, pool)
+	trie, _ := New(common.Hash{}, triedb)
 	updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
 	updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
-	root, _ := trie.Commit()
+	root, _ := trie.Commit(nil)
+	if !memonly {
+		triedb.Commit(root, diskdb)
+	}
 
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err := trie.TryGet([]byte("120000"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err = trie.TryGet([]byte("120099"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err = trie.TryGet([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	err = trie.TryDelete([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
 
 	hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
-	if pooled {
+	if memonly {
-		delete(pool.cache, hash)
+		delete(triedb.nodes, hash)
 	} else {
-		db.Delete(hash[:])
+		diskdb.Delete(hash[:])
 	}
 
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err = trie.TryGet([]byte("120000"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err = trie.TryGet([]byte("120099"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	_, err = trie.TryGet([]byte("123456"))
 	if err != nil {
 		t.Errorf("Unexpected error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
 	}
-	trie, _ = New(root, db, pool)
+	trie, _ = New(root, triedb)
 	err = trie.TryDelete([]byte("123456"))
 	if _, ok := err.(*MissingNodeError); !ok {
 		t.Errorf("Wrong error: %v", err)
@@ -170,7 +170,7 @@ func TestInsert(t *testing.T) {
 	updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 
 	exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
-	root, err := trie.Commit()
+	root, err := trie.Commit(nil)
 	if err != nil {
 		t.Fatalf("commit error: %v", err)
 	}
@@ -199,7 +199,7 @@ func TestGet(t *testing.T) {
 		if i == 1 {
 			return
 		}
-		trie.Commit()
+		trie.Commit(nil)
 	}
 }
 
@@ -268,13 +268,13 @@ func TestReplication(t *testing.T) {
 	for _, val := range vals {
 		updateString(trie, val.k, val.v)
 	}
-	exp, err := trie.Commit()
+	exp, err := trie.Commit(nil)
 	if err != nil {
 		t.Fatalf("commit error: %v", err)
 	}
 
 	// create a new trie on top of the database and check that lookups work.
-	trie2, err := New(exp, trie.db, trie.pool)
+	trie2, err := New(exp, trie.db)
 	if err != nil {
 		t.Fatalf("can't recreate trie at %x: %v", exp, err)
 	}
@@ -283,7 +283,7 @@ func TestReplication(t *testing.T) {
 			t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
 		}
 	}
-	hash, err := trie2.Commit()
+	hash, err := trie2.Commit(nil)
 	if err != nil {
 		t.Fatalf("commit error: %v", err)
 	}
@@ -319,13 +319,13 @@ func TestLargeValue(t *testing.T) {
 }
 
 type countingDB struct {
-	Database
+	DatabaseReader
 	gets map[string]int
 }
 
 func (db *countingDB) Get(key []byte) ([]byte, error) {
 	db.gets[string(key)]++
-	return db.Database.Get(key)
+	return db.DatabaseReader.Get(key)
 }
 
 // TestCacheUnload checks that decoded nodes are unloaded after a
@@ -337,19 +337,20 @@ func TestCacheUnload(t *testing.T) {
 	key2 := "---some other branch"
 	updateString(trie, key1, "this is the branch of key1.")
 	updateString(trie, key2, "this is the branch of key2.")
-	root, _ := trie.Commit()
+
+	root, _ := trie.Commit(nil)
+	trie.db.Commit(root, trie.db.diskdb.(DatabaseWriter))
 
 	// Commit the trie repeatedly and access key1.
 	// The branch containing it is loaded from DB exactly two times:
 	// in the 0th and 6th iteration.
-	db := &countingDB{Database: trie.db, gets: make(map[string]int)}
+	db := &countingDB{DatabaseReader: trie.db.diskdb, gets: make(map[string]int)}
-	trie, _ = New(root, db, trie.pool)
+	trie, _ = New(root, NewDatabase(db))
 	trie.SetCacheLimit(5)
 	for i := 0; i < 12; i++ {
 		getString(trie, key1)
-		trie.Commit()
+		trie.Commit(nil)
 	}
-
 	// Check that it got loaded two times.
 	for dbkey, count := range db.gets {
 		if count != 2 {
@@ -412,10 +413,10 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
 }
 
 func runRandTest(rt randTest) bool {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	mp := NewNodePool()
+	triedb := NewDatabase(diskdb)
 
-	tr, _ := New(common.Hash{}, db, mp)
+	tr, _ := New(common.Hash{}, triedb)
 	values := make(map[string]string) // tracks content of the trie
 
 	for i, step := range rt {
@@ -433,23 +434,23 @@ func runRandTest(rt randTest) bool {
 				rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want)
 			}
 		case opCommit:
-			_, rt[i].err = tr.Commit()
+			_, rt[i].err = tr.Commit(nil)
 		case opHash:
 			tr.Hash()
 		case opReset:
-			hash, err := tr.Commit()
+			hash, err := tr.Commit(nil)
 			if err != nil {
 				rt[i].err = err
 				return false
 			}
-			newtr, err := New(hash, db, mp)
+			newtr, err := New(hash, triedb)
 			if err != nil {
 				rt[i].err = err
 				return false
 			}
 			tr = newtr
 		case opItercheckhash:
-			checktr, _ := New(common.Hash{}, nil, nil)
+			checktr, _ := New(common.Hash{}, triedb)
 			it := NewIterator(tr.NodeIterator(nil))
 			for it.Next() {
 				checktr.Update(it.Key, it.Value)
@@ -522,7 +523,7 @@ func benchGet(b *testing.B, commit bool) {
 	trie := new(Trie)
 	if commit {
 		_, tmpdb := tempDB()
-		trie, _ = New(common.Hash{}, tmpdb, nil)
+		trie, _ = New(common.Hash{}, tmpdb)
 	}
 	k := make([]byte, 32)
 	for i := 0; i < benchElemCount; i++ {
@@ -531,7 +532,7 @@ func benchGet(b *testing.B, commit bool) {
 	}
 	binary.LittleEndian.PutUint64(k, benchElemCount/2)
 	if commit {
-		trie.Commit()
+		trie.Commit(nil)
 	}
 
 	b.ResetTimer()
@@ -541,7 +542,7 @@ func benchGet(b *testing.B, commit bool) {
 	b.StopTimer()
 
 	if commit {
-		ldb := trie.db.(*ethdb.LDBDatabase)
+		ldb := trie.db.diskdb.(*ethdb.LDBDatabase)
 		ldb.Close()
 		os.RemoveAll(ldb.Path())
 	}
@@ -592,16 +593,16 @@ func BenchmarkHash(b *testing.B) {
 	trie.Hash()
 }
 
-func tempDB() (string, Database) {
+func tempDB() (string, *Database) {
 	dir, err := ioutil.TempDir("", "trie-bench")
 	if err != nil {
 		panic(fmt.Sprintf("can't create temporary directory: %v", err))
 	}
-	db, err := ethdb.NewLDBDatabase(dir, 256, 0)
+	diskdb, err := ethdb.NewLDBDatabase(dir, 256, 0)
 	if err != nil {
 		panic(fmt.Sprintf("can't create temporary database: %v", err))
 	}
-	return dir, db
+	return dir, NewDatabase(diskdb)
 }
 
 func getString(trie *Trie, k string) []byte {