forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-17 09:23:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core, trie: minor cleanups, test fixes

Browse source
This commit is contained in:
Péter Szilágyi 2018-01-12 14:38:13 +02:00
parent 0e91a178b7
commit d2ec96e861
No known key found for this signature in database
GPG key ID: E9AE538CEDF8293D
17 changed files with 255 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
core/blockchain.go
View file

@ -96,7 +96,6 @@ type BlockChain struct {
currentBlock *types.Block // Current head of the block chain
currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
trieMemPool *trie.MemPool // Trie node memory pool to avoid storing everything to disk
stateCache state.Database // State database to reuse between imports (contains state cache)
bodyCache *lru.Cache // Cache for the most recent block bodies
bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format
@ -121,7 +120,6 @@ type BlockChain struct {
// available in the database. It initialises the default Ethereum Validator and
// Processor.
func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
trieMemPool := trie.NewMemPool()
bodyCache, _ := lru.New(bodyCacheLimit)
bodyRLPCache, _ := lru.New(bodyCacheLimit)
blockCache, _ := lru.New(blockCacheLimit)
@ -131,8 +129,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
bc := &BlockChain{
config: config,
chainDb: chainDb,
trieMemPool: trieMemPool,
stateCache: state.NewDatabase(chainDb, trieMemPool),
stateCache: state.NewDatabase(chainDb, trie.NewNodePool()),
quit: make(chan struct{}),
bodyCache: bodyCache,
bodyRLPCache: bodyRLPCache,
@ -597,7 +594,7 @@ func (bc *BlockChain) Stop() {
root := bc.CurrentHeader().Root
batch := bc.chainDb.NewBatch()
if err := bc.trieMemPool.Commit(root, batch); err != nil {
if err := bc.stateCache.NodePool().Commit(root, batch); err != nil {
log.Error("Failed to commit latest state trie", "err", err)
}
if err := batch.Write(); err != nil {
@ -809,13 +806,14 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
if err != nil {
return NonStatTy, err
}
bc.trieMemPool.Reference(root, common.Hash{})
pool := bc.stateCache.NodePool()
pool.Reference(root, common.Hash{}) // metadata reference to keep trie alive
if number := block.NumberU64(); number > 192 {
if (number-192)%128 == 0 {
bc.trieMemPool.Commit(root, batch)
pool.Commit(root, batch)
}
header := bc.GetHeaderByNumber(block.NumberU64() - 192)
bc.trieMemPool.Dereference(header.Root, common.Hash{})
pool.Dereference(header.Root, common.Hash{})
}
if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
return NonStatTy, err

8
core/dao_test.go
View file

@ -79,7 +79,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.trieMemPool.Commit(bc.CurrentHeader().Root, db); err != nil {
if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@ -104,7 +104,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.trieMemPool.Commit(bc.CurrentHeader().Root, db); err != nil {
if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@ -130,7 +130,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.trieMemPool.Commit(bc.CurrentHeader().Root, db); err != nil {
if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
@ -150,7 +150,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.trieMemPool.Commit(bc.CurrentHeader().Root, db); err != nil {
if err := bc.stateCache.NodePool().Commit(bc.CurrentHeader().Root, db); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})

33
core/state/database.go
View file

@ -46,14 +46,17 @@ type Database interface {
// OpenStorageTrie opens the storage trie of an account.
OpenStorageTrie(addrHash, root common.Hash) (Trie, error)
// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
// ContractCode retrieves a particular contract's code.
ContractCode(addrHash, codeHash common.Hash) ([]byte, error)
// ContractCode retrieves a particular contracts code's size.
// ContractCodeSize retrieves a particular contracts code's size.
ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
// NodePool retrieves any intermediate trie-node caching layer.
NodePool() *trie.NodePool
}
// Trie is a Ethereum Merkle Trie.
@ -68,20 +71,23 @@ type Trie interface {
}
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains cached trie nodes in memory.
func NewDatabase(db ethdb.Database, pool *trie.MemPool) Database {
// concurrent use and retains cached trie nodes in memory. The pool is an optional
// intermediate trie-node memory pool between the low level storage layer and the
// high level trie abstraction.
func NewDatabase(db ethdb.Database, pool *trie.NodePool) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{db: db, pool: pool, codeSizeCache: csc}
return &cachingDB{db: db, pastNodes: pool, codeSizeCache: csc}
}
type cachingDB struct {
db ethdb.Database
pool *trie.MemPool
mu sync.Mutex
pastTries []*trie.SecureTrie
pastNodes *trie.NodePool
codeSizeCache *lru.Cache
}
// OpenTrie opens the main account trie.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
db.mu.Lock()
defer db.mu.Unlock()
@ -91,7 +97,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
return cachedTrie{db.pastTries[i].Copy(), db}, nil
}
}
tr, err := trie.NewSecure(root, db.db, db.pool, MaxTrieCacheGen)
tr, err := trie.NewSecure(root, db.db, db.pastNodes, MaxTrieCacheGen)
if err != nil {
return nil, err
}
@ -110,10 +116,12 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
}
}
// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
return trie.NewSecure(root, db.db, db.pool, 0)
return trie.NewSecure(root, db.db, db.pastNodes, 0)
}
// CopyTrie returns an independent copy of the given trie.
func (db *cachingDB) CopyTrie(t Trie) Trie {
switch t := t.(type) {
case cachedTrie:
@ -125,6 +133,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
}
}
// ContractCode retrieves a particular contract's code.
func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
code, err := db.db.Get(codeHash[:])
if err == nil {
@ -133,6 +142,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error
return code, err
}
// ContractCodeSize retrieves a particular contracts code's size.
func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) {
if cached, ok := db.codeSizeCache.Get(codeHash); ok {
return cached.(int), nil
@ -144,6 +154,11 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
return len(code), err
}
// NodePool retrieves any intermediate trie-node caching layer.
func (db *cachingDB) NodePool() *trie.NodePool {
return db.pastNodes
}
// cachedTrie inserts its trie into a cachingDB on commit.
type cachedTrie struct {
*trie.SecureTrie

14
core/state/iterator_test.go
View file

@ -39,11 +39,17 @@ func TestNodeIteratorCoverage(t *testing.T) {
hashes[it.Hash] = struct{}{}
}
}
// Cross check the hashes and the database itself
// Cross check the iterated hashes and the database/nodepool content
for hash := range hashes {
if _, err := mem.Get(hash.Bytes()); err != nil {
t.Errorf("failed to retrieve reported node %x: %v", hash, err)
if db.NodePool().Fetch(hash) == nil {
if _, err := mem.Get(hash.Bytes()); err != nil {
t.Errorf("failed to retrieve reported node %x", hash)
}
}
}
for _, hash := range db.NodePool().Nodes() {
if _, ok := hashes[hash]; !ok {
t.Errorf("state entry not reported %x", hash)
}
}
for _, key := range mem.Keys() {

2
core/state/managed_state_test.go
View file

@ -28,7 +28,7 @@ var addr = common.BytesToAddress([]byte("test"))
func create() (*ManagedState, *account) {
db, _ := ethdb.NewMemDatabase()
statedb, _ := New(common.Hash{}, NewDatabase(db, trie.NewMemPool()))
statedb, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
ms := ManageState(statedb)
ms.StateDB.SetNonce(addr, 100)
ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr))

4
core/state/state_test.go
View file

@ -89,7 +89,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
func (s *StateSuite) SetUpTest(c *checker.C) {
s.db, _ = ethdb.NewMemDatabase()
s.state, _ = New(common.Hash{}, NewDatabase(s.db, trie.NewMemPool()))
s.state, _ = New(common.Hash{}, NewDatabase(s.db, trie.NewNodePool()))
}
func (s *StateSuite) TestNull(c *checker.C) {
@ -135,7 +135,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) {
// printing/logging in tests (-check.vv does not work)
func TestSnapshot2(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
state, _ := New(common.Hash{}, NewDatabase(db, trie.NewMemPool()))
state, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
stateobjaddr0 := toAddr([]byte("so0"))
stateobjaddr1 := toAddr([]byte("so1"))

12
core/state/statedb_test.go
View file

@ -41,7 +41,7 @@ import (
func TestUpdateLeaks(t *testing.T) {
// Create an empty state database
db, _ := ethdb.NewMemDatabase()
state, _ := New(common.Hash{}, NewDatabase(db, trie.NewMemPool()))
state, _ := New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
// Update it with some accounts
for i := byte(0); i < 255; i++ {
@ -69,8 +69,8 @@ func TestIntermediateLeaks(t *testing.T) {
// Create two state databases, one transitioning to the final state, the other final from the beginning
transDb, _ := ethdb.NewMemDatabase()
finalDb, _ := ethdb.NewMemDatabase()
transState, _ := New(common.Hash{}, NewDatabase(transDb, trie.NewMemPool()))
finalState, _ := New(common.Hash{}, NewDatabase(finalDb, trie.NewMemPool()))
transState, _ := New(common.Hash{}, NewDatabase(transDb, trie.NewNodePool()))
finalState, _ := New(common.Hash{}, NewDatabase(finalDb, trie.NewNodePool()))
modify := func(state *StateDB, addr common.Address, i, tweak byte) {
state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@ -124,7 +124,7 @@ func TestIntermediateLeaks(t *testing.T) {
func TestCopy(t *testing.T) {
// Create a random state test to copy and modify "independently"
mem, _ := ethdb.NewMemDatabase()
orig, _ := New(common.Hash{}, NewDatabase(mem, trie.NewMemPool()))
orig, _ := New(common.Hash{}, NewDatabase(mem, trie.NewNodePool()))
for i := byte(0); i < 255; i++ {
obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@ -336,8 +336,8 @@ func (test *snapshotTest) run() bool {
// Run all actions and create snapshots.
var (
db, _ = ethdb.NewMemDatabase()
mp = trie.NewMemPool()
state, _ = New(common.Hash{}, NewDatabase(db, trie.NewMemPool()))
mp = trie.NewNodePool()
state, _ = New(common.Hash{}, NewDatabase(db, trie.NewNodePool()))
snapshotRevs = make([]int, len(test.snapshots))
sindex = 0
)

73
core/state/sync_test.go
View file

@ -39,7 +39,7 @@ type testAccount struct {
func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) {
// Create an empty state
mem, _ := ethdb.NewMemDatabase()
db := NewDatabase(mem, trie.NewMemPool())
db := NewDatabase(mem, trie.NewNodePool())
state, _ := New(common.Hash{}, db)
// Fill it with some arbitrary data
@ -71,7 +71,7 @@ func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount)
// account array.
func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
// Check root availability and state contents
state, err := New(root, NewDatabase(db, trie.NewMemPool()))
state, err := New(root, NewDatabase(db, trie.NewNodePool()))
if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err)
}
@ -96,7 +96,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
if v, _ := db.Get(root[:]); v == nil {
return nil // Consider a non existent state consistent.
}
trie, err := trie.New(root, db, trie.NewMemPool())
trie, err := trie.New(root, db, trie.NewNodePool())
if err != nil {
return err
}
@ -112,7 +112,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
if _, err := db.Get(root.Bytes()); err != nil {
return nil // Consider a non existent state consistent.
}
state, err := New(root, NewDatabase(db, trie.NewMemPool()))
state, err := New(root, NewDatabase(db, trie.NewNodePool()))
if err != nil {
return err
}
@ -138,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t,
func testIterativeStateSync(t *testing.T, batch int) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -148,9 +148,14 @@ func testIterativeStateSync(t *testing.T, batch int) {
for len(queue) > 0 {
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcMem.Get(hash.Bytes())
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
var (
data = srcDb.NodePool().Fetch(hash)
err error
)
if data == nil {
if data, err = srcMem.Get(hash.Bytes()); err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
@ -170,7 +175,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
// partial results are returned, and the others sent only later.
func TestIterativeDelayedStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -181,9 +186,14 @@ func TestIterativeDelayedStateSync(t *testing.T) {
// Sync only half of the scheduled nodes
results := make([]trie.SyncResult, len(queue)/2+1)
for i, hash := range queue[:len(results)] {
data, err := srcMem.Get(hash.Bytes())
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
var (
data = srcDb.NodePool().Fetch(hash)
err error
)
if data == nil {
if data, err = srcMem.Get(hash.Bytes()); err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
@ -207,7 +217,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomS
func testIterativeRandomStateSync(t *testing.T, batch int) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -221,9 +231,14 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
// Fetch all the queued nodes in a random order
results := make([]trie.SyncResult, 0, len(queue))
for hash := range queue {
data, err := srcMem.Get(hash.Bytes())
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
var (
data = srcDb.NodePool().Fetch(hash)
err error
)
if data == nil {
if data, err = srcMem.Get(hash.Bytes()); err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
}
results = append(results, trie.SyncResult{Hash: hash, Data: data})
}
@ -247,7 +262,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
// partial results are returned (Even those randomly), others sent only later.
func TestIterativeRandomDelayedStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -263,9 +278,14 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for hash := range queue {
delete(queue, hash)
data, err := srcMem.Get(hash.Bytes())
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
var (
data = srcDb.NodePool().Fetch(hash)
err error
)
if data == nil {
if data, err = srcMem.Get(hash.Bytes()); err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
}
results = append(results, trie.SyncResult{Hash: hash, Data: data})
@ -292,7 +312,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
// the database.
func TestIncompleteStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcMem, srcRoot, srcAccounts := makeTestState()
checkTrieConsistency(srcMem, srcRoot)
@ -306,9 +326,14 @@ func TestIncompleteStateSync(t *testing.T) {
// Fetch a batch of state nodes
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcMem.Get(hash.Bytes())
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
var (
data = srcDb.NodePool().Fetch(hash)
err error
)
if data == nil {
if data, err = srcMem.Get(hash.Bytes()); err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}

4
eth/downloader/downloader_test.go
View file

@ -293,7 +293,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
// For now only check that the state trie is correct
if block := dl.GetBlockByHash(hash); block != nil {
_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
_, err := trie.NewSecure(block.Root(), dl.stateDb, nil, 0) // nil trie node-cache, ensure we have everything on disk
return err
}
return fmt.Errorf("non existent block: %x", hash[:4])
@ -660,7 +660,7 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
}
if index > 0 {
if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb)); statedb == nil || err != nil {
if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb, nil)); statedb == nil || err != nil {
t.Fatalf("state reconstruction failed: %v", err)
}
}

4
light/trie.go
View file

@ -83,6 +83,10 @@ func (db *odrDatabase) ContractCodeSize(addrHash, codeHash common.Hash) (int, er
return len(code), err
}
func (db *odrDatabase) NodePool() *trie.NodePool {
return nil
}
type odrTrie struct {
db *odrDatabase
id *TrieID

9
trie/hasher.go
View file

@ -52,7 +52,7 @@ func returnHasherToPool(h *hasher) {
// hash collapses a node down into a hash node, also returning a copy of the
// original node initialized with the computed hash to replace the original one.
func (h *hasher) hash(n node, pool *MemPool, force bool) (node, node, error) {
func (h *hasher) hash(n node, pool *NodePool, force bool) (node, node, error) {
// If we're not storing the node, just hashing, use available cached data
if hash, dirty := n.cache(); hash != nil {
if pool == nil {
@ -99,7 +99,7 @@ func (h *hasher) hash(n node, pool *MemPool, force bool) (node, node, error) {
// hashChildren replaces the children of a node with their hashes if the encoded
// size of the child is larger than a hash, returning the collapsed node as well
// as a replacement for the original node with the child hashes cached in.
func (h *hasher) hashChildren(original node, pool *MemPool) (node, node, []common.Hash, error) {
func (h *hasher) hashChildren(original node, pool *NodePool) (node, node, []common.Hash, error) {
var err error
switch n := original.(type) {
@ -181,7 +181,10 @@ func (h *hasher) externals(n node) []common.Hash {
return []common.Hash{account.Root}
}
func (h *hasher) store(n node, refs []common.Hash, pool *MemPool, force bool) (node, []common.Hash, error) {
// store hashes the node n and if we have a storage layer specified, it writes
// the key/value pair to it and tracks any node->child references as well as any
// node->external trie references.
func (h *hasher) store(n node, refs []common.Hash, pool *NodePool, force bool) (node, []common.Hash, error) {
// Don't store hashes or empty nodes.
if _, isHash := n.(hashNode); n == nil || isHash {
return n, refs, nil

100
trie/iterator_test.go
View file

@ -278,45 +278,74 @@ func TestIteratorNoDups(t *testing.T) {
}
// This test checks that nodeIterator.Next can be retried after inserting missing trie nodes.
func TestIteratorContinueAfterError(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
mp := NewMemPool()
func TestIteratorContinueAfterErrorDirect(t *testing.T) { testIteratorContinueAfterError(t, false) }
func TestIteratorContinueAfterErrorPooled(t *testing.T) { testIteratorContinueAfterError(t, true) }
tr, _ := New(common.Hash{}, db, mp)
func testIteratorContinueAfterError(t *testing.T, pooled bool) {
var pool *NodePool
if pooled {
pool = NewNodePool()
}
db, _ := ethdb.NewMemDatabase()
tr, _ := New(common.Hash{}, db, pool)
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
tr.Commit()
wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
keys := db.Keys()
t.Log("node count", wantNodeCount)
var (
dbKeys [][]byte
poolKeys []common.Hash
)
if pooled {
poolKeys = pool.Nodes()
} else {
dbKeys = db.Keys()
}
for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB.
tr, _ := New(tr.Hash(), db, mp)
tr, _ := New(tr.Hash(), db, pool)
// Remove a random node from the database. It can't be the root node
// because that one is already loaded.
var rkey []byte
var (
rkey common.Hash
rval []byte
)
for {
if rkey = keys[rand.Intn(len(keys))]; !bytes.Equal(rkey, tr.Hash().Bytes()) {
if pooled {
rkey = poolKeys[rand.Intn(len(poolKeys))]
} else {
copy(rkey[:], dbKeys[rand.Intn(len(dbKeys))])
}
if rkey != tr.Hash() {
break
}
}
rval, _ := db.Get(rkey)
db.Delete(rkey)
if pooled {
rval, _ = pool.cache[rkey]
delete(pool.cache, rkey)
} else {
rval, _ = db.Get(rkey[:])
db.Delete(rkey[:])
}
// Iterate until the error is hit.
seen := make(map[string]bool)
it := tr.NodeIterator(nil)
checkIteratorNoDups(t, it, seen)
missing, ok := it.Error().(*MissingNodeError)
if !ok || !bytes.Equal(missing.NodeHash[:], rkey) {
if !ok || missing.NodeHash != rkey {
t.Fatal("didn't hit missing node, got", it.Error())
}
// Add the node back and continue iteration.
db.Put(rkey, rval)
if pooled {
pool.cache[rkey] = rval
} else {
db.Put(rkey[:], rval)
}
checkIteratorNoDups(t, it, seen)
if it.Error() != nil {
t.Fatal("unexpected error", it.Error())
@ -330,23 +359,40 @@ func TestIteratorContinueAfterError(t *testing.T) {
// Similar to the test above, this one checks that failure to create nodeIterator at a
// certain key prefix behaves correctly when Next is called. The expectation is that Next
// should retry seeking before returning true for the first time.
func TestIteratorContinueAfterSeekError(t *testing.T) {
// Commit test trie to db, then remove the node containing "bars".
db, _ := ethdb.NewMemDatabase()
mp := NewMemPool()
func TestIteratorContinueAfterSeekErrorDirect(t *testing.T) {
testIteratorContinueAfterSeekError(t, false)
}
func TestIteratorContinueAfterSeekErrorPooled(t *testing.T) {
testIteratorContinueAfterSeekError(t, true)
}
ctr, _ := New(common.Hash{}, db, mp)
func testIteratorContinueAfterSeekError(t *testing.T, pooled bool) {
// Commit test trie to db, then remove the node containing "bars".
var pool *NodePool
if pooled {
pool = NewNodePool()
}
db, _ := ethdb.NewMemDatabase()
ctr, _ := New(common.Hash{}, db, pool)
for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v))
}
root, _ := ctr.Commit()
barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
barNode, _ := db.Get(barNodeHash[:])
db.Delete(barNodeHash[:])
barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
var barNodeBlob []byte
if pooled {
barNodeBlob = pool.cache[barNodeHash]
delete(pool.cache, barNodeHash)
} else {
barNodeBlob, _ = db.Get(barNodeHash[:])
db.Delete(barNodeHash[:])
}
// Create a new iterator that seeks to "bars". Seeking can't proceed because
// the node is missing.
tr, _ := New(root, db, mp)
tr, _ := New(root, db, pool)
it := tr.NodeIterator([]byte("bars"))
missing, ok := it.Error().(*MissingNodeError)
if !ok {
@ -354,10 +400,12 @@ func TestIteratorContinueAfterSeekError(t *testing.T) {
} else if missing.NodeHash != barNodeHash {
t.Fatal("wrong node missing")
}
// Reinsert the missing node.
db.Put(barNodeHash[:], barNode[:])
if pooled {
pool.cache[barNodeHash] = barNodeBlob
} else {
db.Put(barNodeHash[:], barNodeBlob)
}
// Check that iteration produces the right set of values.
if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {
t.Fatal(err)

40
trie/mempool.go → trie/node_pool.go
View file

@ -24,10 +24,10 @@ import (
"github.com/ethereum/go-ethereum/log"
)
// MemPool is an intermediate write layer between the trie data structures and
// NodePool is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
type MemPool struct {
type NodePool struct {
cache map[common.Hash][]byte // Cached data blocks of the trie nodes
parents map[common.Hash]int // Number of live nodes referencing a given one
@ -41,10 +41,10 @@ type MemPool struct {
lock sync.RWMutex
}
// NewMemPool creates a new memory pool to store ephemeral trie nodes before they
// NewNodePool creates a new memory pool to store ephemeral trie nodes before they
// are written out to disk or garbage collected.
func NewMemPool() *MemPool {
pool := &MemPool{
func NewNodePool() *NodePool {
pool := &NodePool{
cache: make(map[common.Hash][]byte),
parents: make(map[common.Hash]int),
children: make(map[common.Hash]map[common.Hash]struct{}),
@ -57,7 +57,7 @@ func NewMemPool() *MemPool {
// will make a copy of the slice.
//
// Note, this method assumes that the pool's lock is held!
func (pool *MemPool) insert(hash common.Hash, blob []byte) {
func (pool *NodePool) insert(hash common.Hash, blob []byte) {
if _, ok := pool.cache[hash]; ok {
return
}
@ -69,15 +69,29 @@ func (pool *MemPool) insert(hash common.Hash, blob []byte) {
// Fetch retrieves a cached trie node from memory, or returns nil if the pool
// does not have this particular piece of data.
func (pool *MemPool) Fetch(hash common.Hash) []byte {
func (pool *NodePool) Fetch(hash common.Hash) []byte {
pool.lock.RLock()
defer pool.lock.RUnlock()
return pool.cache[hash]
}
// Nodes retrieves the hashes of all the nodes cached within the node pool. This
// method is extremely expensive and should only be used in test code to validate
// internal states.
func (pool *NodePool) Nodes() []common.Hash {
pool.lock.RLock()
defer pool.lock.RUnlock()
var hashes = make([]common.Hash, 0, len(pool.cache))
for hash := range pool.cache {
hashes = append(hashes, hash)
}
return hashes
}
// Reference adds a new reference from parent to node.
func (pool *MemPool) Reference(node common.Hash, parent common.Hash) {
func (pool *NodePool) Reference(node common.Hash, parent common.Hash) {
pool.lock.RLock()
defer pool.lock.RUnlock()
@ -85,7 +99,7 @@ func (pool *MemPool) Reference(node common.Hash, parent common.Hash) {
}
// reference is the private locked version of Reference.
func (pool *MemPool) reference(node common.Hash, parent common.Hash) {
func (pool *NodePool) reference(node common.Hash, parent common.Hash) {
// If the node does not exist, it's a node pulled from disk, skip
if _, ok := pool.cache[node]; !ok {
return
@ -95,7 +109,7 @@ func (pool *MemPool) reference(node common.Hash, parent common.Hash) {
}
// Dereference removes an existing reference from parent to node.
func (pool *MemPool) Dereference(node common.Hash, parent common.Hash) {
func (pool *NodePool) Dereference(node common.Hash, parent common.Hash) {
pool.lock.Lock()
defer pool.lock.Unlock()
@ -108,7 +122,7 @@ func (pool *MemPool) Dereference(node common.Hash, parent common.Hash) {
}
// dereference is the private locked version of Dereference.
func (pool *MemPool) dereference(node common.Hash, parent common.Hash) {
func (pool *NodePool) dereference(node common.Hash, parent common.Hash) {
// If the node does not exist, it's a previously comitted node.
blob, ok := pool.cache[node]
if !ok {
@ -132,7 +146,7 @@ func (pool *MemPool) dereference(node common.Hash, parent common.Hash) {
// Commit iterates over all the children of a particular node, writes them out
// to disk, forcefully tearing down all references in both directions.
func (pool *MemPool) Commit(node common.Hash, db DatabaseWriter) error {
func (pool *NodePool) Commit(node common.Hash, db DatabaseWriter) error {
pool.lock.Lock()
defer pool.lock.Unlock()
@ -158,7 +172,7 @@ func (pool *MemPool) Commit(node common.Hash, db DatabaseWriter) error {
}
// commit is the private locked version of Commit.
func (pool *MemPool) commit(node common.Hash, db DatabaseWriter) error {
func (pool *NodePool) commit(node common.Hash, db DatabaseWriter) error {
// If the node does not exist, it's a previously comitted node.
blob, ok := pool.cache[node]
if !ok {

7
trie/secure_trie.go
View file

@ -45,17 +45,18 @@ type SecureTrie struct {
secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
}
// NewSecure creates a trie with an existing root node from db.
// NewSecure creates a trie with an existing root node from a backing database
// and optional intermediate in-memory node pool.
//
// If root is the zero hash or the sha3 hash of an empty string, the
// trie is initially empty. Otherwise, New will panic if db is nil
// and returns MissingNodeError if the root node cannot be found.
//
// Accessing the trie loads nodes from db on demand.
// Accessing the trie loads nodes from the database or node pool on demand.
// Loaded nodes are kept around until their 'cache generation' expires.
// A new cache generation is created by each call to Commit.
// cachelimit sets the number of past cache generations to keep.
func NewSecure(root common.Hash, db Database, pool *MemPool, cachelimit uint16) (*SecureTrie, error) {
func NewSecure(root common.Hash, db Database, pool *NodePool, cachelimit uint16) (*SecureTrie, error) {
if db == nil {
panic("NewSecure called with nil database")
}

4
trie/secure_trie_test.go
View file

@ -29,7 +29,7 @@ import (
func newEmptySecure() *SecureTrie {
db, _ := ethdb.NewMemDatabase()
trie, _ := NewSecure(common.Hash{}, db, NewMemPool(), 0)
trie, _ := NewSecure(common.Hash{}, db, NewNodePool(), 0)
return trie
}
@ -37,7 +37,7 @@ func newEmptySecure() *SecureTrie {
func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
// Create an empty trie
db, _ := ethdb.NewMemDatabase()
trie, _ := NewSecure(common.Hash{}, db, NewMemPool(), 0)
trie, _ := NewSecure(common.Hash{}, db, NewNodePool(), 0)
// Fill it with some arbitrary data
content := make(map[string][]byte)

10
trie/trie.go
View file

@ -85,7 +85,7 @@ type DatabaseWriter interface {
type Trie struct {
root node
db Database
pool *MemPool
pool *NodePool
originalRoot common.Hash
// Cache generation values.
@ -112,7 +112,7 @@ func (t *Trie) newFlag() nodeFlag {
// trie is initially empty and does not require a database. Otherwise,
// New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand.
func New(root common.Hash, db Database, pool *MemPool) (*Trie, error) {
func New(root common.Hash, db Database, pool *NodePool) (*Trie, error) {
trie := &Trie{db: db, pool: pool, originalRoot: root}
if (root != common.Hash{}) && root != emptyRoot {
if db == nil {
@ -496,12 +496,12 @@ func (t *Trie) Commit() (root common.Hash, err error) {
// database before using the trie.
func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
// Retrieve the intermedia trie node memory cache if really writing
var pool *MemPool
var pool *NodePool
if db != nil {
if pool = t.pool; pool == nil {
// If the trie has no intermediate memory pool, but actual database write was
// nonetheless requested, store into an emphemeral pool and flush out to disk.
pool = NewMemPool()
pool = NewNodePool()
defer func() {
for hash, blob := range pool.cache {
db.Put(hash[:], blob)
@ -519,7 +519,7 @@ func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
return common.BytesToHash(hash.(hashNode)), nil
}
func (t *Trie) hashRoot(pool *MemPool) (node, node, error) {
func (t *Trie) hashRoot(pool *NodePool) (node, node, error) {
if t.root == nil {
return hashNode(emptyRoot.Bytes()), nil, nil
}

55
trie/trie_test.go
View file

@ -44,7 +44,7 @@ func init() {
// Used for testing
func newEmpty() *Trie {
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db, NewMemPool())
trie, _ := New(common.Hash{}, db, NewNodePool())
return trie
}
@ -69,7 +69,7 @@ func TestNull(t *testing.T) {
func TestMissingRoot(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db, NewMemPool())
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db, NewNodePool())
if trie != nil {
t.Error("New returned non-nil trie for invalid root")
}
@ -78,72 +78,75 @@ func TestMissingRoot(t *testing.T) {
}
}
func TestMissingNode(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
mp := NewMemPool()
func TestMissingNodeDirect(t *testing.T) { testMissingNode(t, false) }
func TestMissingNodePooled(t *testing.T) { testMissingNode(t, true) }
trie, _ := New(common.Hash{}, db, mp)
func testMissingNode(t *testing.T, pooled bool) {
var pool *NodePool
if pooled {
pool = NewNodePool()
}
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db, pool)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _ := trie.Commit()
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err := trie.TryGet([]byte("120000"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err = trie.TryGet([]byte("120099"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
err = trie.TryDelete([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
db.Delete(common.FromHex("e1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9"))
hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9")
if pooled {
delete(pool.cache, hash)
} else {
db.Delete(hash[:])
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err = trie.TryGet([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err = trie.TryGet([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db, mp)
trie, _ = New(root, db, pool)
err = trie.TryDelete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
@ -410,7 +413,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
func runRandTest(rt randTest) bool {
db, _ := ethdb.NewMemDatabase()
mp := NewMemPool()
mp := NewNodePool()
tr, _ := New(common.Hash{}, db, mp)
values := make(map[string]string) // tracks content of the trie