forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 06:36:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

core, light, miner, trie: add direct cache for state values

Browse source
This commit is contained in:
Nick Johnson 2016-10-21 15:00:20 +01:00 committed by Péter Szilágyi
parent 9f8bc00cf5
commit d679842f1c
No known key found for this signature in database
GPG key ID: 119A76381CCB7DD2
23 changed files with 422 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
cmd/geth/chaincmd.go
View file

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/syndtr/goleveldb/leveldb/util"
"gopkg.in/urfave/cli.v1"
@ -287,6 +288,49 @@ func dump(ctx *cli.Context) error {
return nil
}
var accountCacheKeyPrefix = []byte("accounthashcache:")
type cachedAccount struct {
state.Account
BlockNum uint64
BlockHash common.Hash
}
func buildCache(ctx *cli.Context) error {
stack := makeFullNode(ctx)
chain, chainDb := utils.MakeChain(ctx, stack)
defer chainDb.Close()
num, _ := strconv.Atoi(ctx.Args()[0])
blockNum := uint64(num)
block := chain.GetBlockByNumber(blockNum)
blockHash := block.Hash()
t, err := trie.New(block.Root(), chainDb, 0)
if err != nil {
utils.Fatalf("Could not open trie: %v", err)
}
st := trie.NewSecure(t, chainDb)
iter := st.Iterator()
i := 0
for iter.Next() {
var data state.Account
if err := rlp.DecodeBytes(iter.Value, &data); err != nil {
utils.Fatalf("can't decode object at %x: %v", iter.Key[:], err)
}
enc, _ := rlp.EncodeToBytes(cachedAccount{data, blockNum, blockHash})
if err := chainDb.Put(append(accountCacheKeyPrefix, iter.Key[:]...), enc); err != nil {
utils.Fatalf("Could not write to DB: %v", err)
}
i += 1
if i % 10000 == 0 {
fmt.Printf("Processed %d accounts, at %v\n", i, common.ToHex(iter.Key))
}
}
return nil
}
// hashish returns true for strings that look like hashes.
func hashish(x string) bool {
_, err := strconv.Atoi(x)

7
core/blockchain.go
View file

@ -275,7 +275,7 @@ func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x…]", hash[:4])
}
if _, err := trie.NewSecure(block.Root(), self.chainDb, 0); err != nil {
if _, err := trie.New(block.Root(), self.chainDb, 0); err != nil {
return err
}
// If all checks out, manually set the head block
@ -551,6 +551,11 @@ func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block {
return self.GetBlock(hash, number)
}
// IsCanonChainBlock checks whether the given block is in the current canonical chain.
func (self *BlockChain) IsCanonChainBlock(number uint64, hash common.Hash) bool {
return number < uint64(self.currentBlock.Number().Int64()) && GetCanonicalHash(self.chainDb, number) == hash
}
// [deprecated by eth/62]
// GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.
func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) {

2
core/chain_makers.go
View file

@ -107,7 +107,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
if b.gasPool == nil {
b.SetCoinbase(common.Address{})
}
b.statedb.StartRecord(tx.Hash(), common.Hash{}, len(b.txs))
b.statedb.SetTxContext(common.Hash{}, b.header.Number.Uint64(), tx.Hash(), len(b.txs), nil)
receipt, _, _, err := ApplyTransaction(b.config, nil, b.gasPool, b.statedb, b.header, tx, b.header.GasUsed, vm.Config{})
if err != nil {
panic(err)

8
core/state/dump.go
View file

@ -44,9 +44,9 @@ func (self *StateDB) RawDump() Dump {
Accounts: make(map[string]DumpAccount),
}
it := self.trie.Iterator()
it := self.storage.Iterator()
for it.Next() {
addr := self.trie.GetKey(it.Key)
addr := self.storage.GetKey(it.Key)
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
panic(err)
@ -61,9 +61,9 @@ func (self *StateDB) RawDump() Dump {
Code: common.Bytes2Hex(obj.Code(self.db)),
Storage: make(map[string]string),
}
storageIt := obj.getTrie(self.db).Iterator()
storageIt := obj.getStorage(self.db).Iterator()
for storageIt.Next() {
account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
account.Storage[common.Bytes2Hex(self.storage.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
}
dump.Accounts[common.Bytes2Hex(addr)] = account
}

4
core/state/iterator.go
View file

@ -76,7 +76,7 @@ func (it *NodeIterator) step() error {
}
// Initialize the iterator if we've just started
if it.stateIt == nil {
it.stateIt = it.state.trie.NodeIterator()
it.stateIt = trie.NewNodeIterator(it.state.trie)
}
// If we had data nodes previously, we surely have at least state nodes
if it.dataIt != nil {
@ -115,7 +115,7 @@ func (it *NodeIterator) step() error {
if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob), &account); err != nil {
return err
}
dataTrie, err := trie.New(account.Root, it.state.db)
dataTrie, err := trie.New(account.Root, it.state.db, 0)
if err != nil {
return err
}

2
core/state/iterator_test.go
View file

@ -47,7 +47,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
}
}
for _, key := range db.(*ethdb.MemDatabase).Keys() {
if bytes.HasPrefix(key, []byte("secure-key-")) {
if bytes.HasPrefix(key, []byte("secure-key-")) || bytes.HasPrefix(key, CachePrefix) {
continue
}
if _, ok := hashes[common.BytesToHash(key)]; !ok {

23
core/state/state_object.go
View file

@ -76,7 +76,8 @@ type StateObject struct {
dbErr error
// Write caches.
trie *trie.SecureTrie // storage trie, which becomes non-nil on first access
trie *trie.Trie // storage trie, which becomes non-nil on first access
storage *trie.SecureTrie // Interface to storage
code Code // contract bytecode, which gets set when code is loaded
cachedStorage Storage // Storage entry cache to avoid duplicate reads
@ -152,16 +153,17 @@ func (c *StateObject) touch() {
c.touched = true
}
func (c *StateObject) getTrie(db trie.Database) *trie.SecureTrie {
func (c *StateObject) getStorage(db trie.Database) *trie.SecureTrie {
if c.trie == nil {
var err error
c.trie, err = trie.NewSecure(c.data.Root, db, 0)
c.trie, err = trie.New(c.data.Root, db, 0)
if err != nil {
c.trie, _ = trie.NewSecure(common.Hash{}, db, 0)
c.trie, _ = trie.New(common.Hash{}, nil, 0)
c.setError(fmt.Errorf("can't create storage trie: %v", err))
}
c.storage = trie.NewSecure(c.trie, db)
}
return c.trie
return c.storage
}
// GetState returns a value in account storage.
@ -171,7 +173,7 @@ func (self *StateObject) GetState(db trie.Database, key common.Hash) common.Hash
return value
}
// Load from DB in case it is missing.
if enc := self.getTrie(db).Get(key[:]); len(enc) > 0 {
if enc := self.getStorage(db).Get(key[:]); len(enc) > 0 {
_, content, _, err := rlp.Split(enc)
if err != nil {
self.setError(err)
@ -206,7 +208,7 @@ func (self *StateObject) setState(key, value common.Hash) {
// updateTrie writes cached storage modifications into the object's storage trie.
func (self *StateObject) updateTrie(db trie.Database) {
tr := self.getTrie(db)
tr := self.getStorage(db)
for key, value := range self.dirtyStorage {
delete(self.dirtyStorage, key)
if (value == common.Hash{}) {
@ -232,7 +234,7 @@ func (self *StateObject) CommitTrie(db trie.Database, dbw trie.DatabaseWriter) e
if self.dbErr != nil {
return self.dbErr
}
root, err := self.trie.CommitTo(dbw)
root, err := self.storage.CommitTo(dbw)
if err == nil {
self.data.Root = root
}
@ -293,6 +295,7 @@ func (c *StateObject) ReturnGas(gas, price *big.Int) {}
func (self *StateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *StateObject {
stateObject := newObject(db, self.address, self.data, onDirty)
stateObject.trie = self.trie
stateObject.storage = self.storage
stateObject.code = self.code
stateObject.dirtyStorage = self.dirtyStorage.Copy()
stateObject.cachedStorage = self.dirtyStorage.Copy()
@ -388,10 +391,10 @@ func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) {
cb(h, value)
}
it := self.getTrie(self.db.db).Iterator()
it := self.getStorage(self.db.db).Iterator()
for it.Next() {
// ignore cached values
key := common.BytesToHash(self.trie.GetKey(it.Key))
key := common.BytesToHash(self.storage.GetKey(it.Key))
if _, ok := self.cachedStorage[key]; !ok {
cb(key, common.BytesToHash(it.Value))
}

53
core/state/statedb.go
View file

@ -37,6 +37,8 @@ import (
// Trie cache generation limit after which to evic trie nodes from memory.
var MaxTrieCacheGen = uint16(120)
var CachePrefix = []byte("accounthashcache:")
const (
// Number of past tries to keep. This value is chosen such that
// reasonable chain reorg depths will hit an existing trie.
@ -58,8 +60,9 @@ type revision struct {
// * Accounts
type StateDB struct {
db ethdb.Database
trie *trie.SecureTrie
pastTries []*trie.SecureTrie
trie *trie.Trie
storage *trie.SecureTrie
pastTries []*trie.Trie
codeSizeCache *lru.Cache
// This map holds 'live' objects, which will get modified while processing a state transition.
@ -85,12 +88,13 @@ type StateDB struct {
// Create a new state from a given trie
func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
tr, err := trie.NewSecure(root, db, MaxTrieCacheGen)
tr, err := trie.New(root, db, MaxTrieCacheGen)
if err != nil {
return nil, err
}
csc, _ := lru.New(codeSizeCacheSize)
return &StateDB{
ret := &StateDB{
db: db,
trie: tr,
codeSizeCache: csc,
@ -98,7 +102,9 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
stateObjectsDirty: make(map[common.Address]struct{}),
refund: new(big.Int),
logs: make(map[common.Hash]vm.Logs),
}, nil
}
ret.SetTxContext(common.Hash{}, 0, common.Hash{}, 0, nil)
return ret, nil
}
// New creates a new statedb by reusing any journalled tries to avoid costly
@ -111,7 +117,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
if err != nil {
return nil, err
}
return &StateDB{
ret := &StateDB{
db: self.db,
trie: tr,
codeSizeCache: self.codeSizeCache,
@ -119,7 +125,9 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
stateObjectsDirty: make(map[common.Address]struct{}),
refund: new(big.Int),
logs: make(map[common.Hash]vm.Logs),
}, nil
}
ret.SetTxContext(common.Hash{}, 0, common.Hash{}, 0, nil)
return ret, nil
}
// Reset clears out all emphemeral state objects from the state db, but keeps
@ -141,23 +149,25 @@ func (self *StateDB) Reset(root common.Hash) error {
self.logs = make(map[common.Hash]vm.Logs)
self.logSize = 0
self.clearJournalAndRefund()
self.SetTxContext(common.Hash{}, 0, common.Hash{}, 0, nil)
return nil
}
// openTrie creates a trie. It uses an existing trie if one is available
// from the journal if available.
func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) {
func (self *StateDB) openTrie(root common.Hash) (*trie.Trie, error) {
for i := len(self.pastTries) - 1; i >= 0; i-- {
if self.pastTries[i].Hash() == root {
tr := *self.pastTries[i]
return &tr, nil
}
}
return trie.NewSecure(root, self.db, MaxTrieCacheGen)
return trie.New(root, self.db, MaxTrieCacheGen)
}
func (self *StateDB) pushTrie(t *trie.SecureTrie) {
func (self *StateDB) pushTrie(t *trie.Trie) {
self.lock.Lock()
defer self.lock.Unlock()
@ -169,10 +179,16 @@ func (self *StateDB) pushTrie(t *trie.SecureTrie) {
}
}
func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) {
self.thash = thash
self.bhash = bhash
self.txIndex = ti
func (self *StateDB) SetTxContext(blockHash common.Hash, blockNum uint64, txHash common.Hash, txIndex int, validator trie.CacheValidator) {
self.bhash = blockHash
self.thash = txHash
self.txIndex = txIndex
if validator == nil {
validator = &trie.NullCacheValidator{}
}
storage := trie.NewDirectCache(self.trie, self.db, CachePrefix, validator, true)
self.storage = trie.NewSecure(storage, self.db)
}
func (self *StateDB) AddLog(log *vm.Log) {
@ -359,14 +375,14 @@ func (self *StateDB) updateStateObject(stateObject *StateObject) {
if err != nil {
panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
}
self.trie.Update(addr[:], data)
self.storage.Update(addr[:], data)
}
// deleteStateObject removes the given object from the state trie.
func (self *StateDB) deleteStateObject(stateObject *StateObject) {
stateObject.deleted = true
addr := stateObject.Address()
self.trie.Delete(addr[:])
self.storage.Delete(addr[:])
}
// Retrieve a state object given my the address. Returns nil if not found.
@ -380,7 +396,7 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje
}
// Load the object from the database.
enc := self.trie.Get(addr[:])
enc := self.storage.Get(addr[:])
if len(enc) == 0 {
return nil
}
@ -460,6 +476,7 @@ func (self *StateDB) Copy() *StateDB {
state := &StateDB{
db: self.db,
trie: self.trie,
storage: self.storage,
pastTries: self.pastTries,
codeSizeCache: self.codeSizeCache,
stateObjects: make(map[common.Address]*StateObject, len(self.stateObjectsDirty)),
@ -607,7 +624,7 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root
delete(s.stateObjectsDirty, addr)
}
// Write trie changes.
root, err = s.trie.CommitTo(dbw)
root, err = s.storage.CommitTo(dbw)
if err == nil {
s.pushTrie(s.trie)
}

3
core/state_processor.go
View file

@ -72,8 +72,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
}
// Iterate over and process the individual transactions
for i, tx := range block.Transactions() {
//fmt.Println("tx:", i)
statedb.StartRecord(tx.Hash(), block.Hash(), i)
statedb.SetTxContext(block.Hash(), block.NumberU64(), tx.Hash(), i, p.bc)
receipt, logs, _, err := ApplyTransaction(p.config, p.bc, gp, statedb, header, tx, totalUsedGas, cfg)
if err != nil {
return nil, nil, nil, err

2
core/types/derive_sha.go
View file

@ -31,7 +31,7 @@ type DerivableList interface {
func DeriveSha(list DerivableList) common.Hash {
keybuf := new(bytes.Buffer)
trie := new(trie.Trie)
trie, _ := trie.New(common.Hash{}, nil, 0)
for i := 0; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))

2
eth/downloader/downloader_test.go
View file

@ -294,7 +294,7 @@ func (dl *downloadTester) headFastBlock() *types.Block {
func (dl *downloadTester) commitHeadBlock(hash common.Hash) error {
// For now only check that the state trie is correct
if block := dl.getBlock(hash); block != nil {
_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
_, err := trie.New(block.Root(), dl.stateDb, 0)
return err
}
return fmt.Errorf("non existent block: %x", hash[:4])

8
les/handler.go
View file

@ -637,7 +637,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
for _, req := range req.Reqs {
// Retrieve the requested state entry, stopping if enough was found
if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil {
if trie, _ := trie.New(header.Root, pm.chainDb); trie != nil {
if trie, _ := trie.New(header.Root, pm.chainDb, 0); trie != nil {
sdata := trie.Get(req.AccKey)
var acc state.Account
if err := rlp.DecodeBytes(sdata, &acc); err == nil {
@ -764,13 +764,13 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
// Retrieve the requested state entry, stopping if enough was found
if header := core.GetHeader(pm.chainDb, req.BHash, core.GetBlockNumber(pm.chainDb, req.BHash)); header != nil {
if tr, _ := trie.New(header.Root, pm.chainDb); tr != nil {
if tr, _ := trie.New(header.Root, pm.chainDb, 0); tr != nil {
if len(req.AccKey) > 0 {
sdata := tr.Get(req.AccKey)
tr = nil
var acc state.Account
if err := rlp.DecodeBytes(sdata, &acc); err == nil {
tr, _ = trie.New(acc.Root, pm.chainDb)
tr, _ = trie.New(acc.Root, pm.chainDb, 0)
}
}
if tr != nil {
@ -832,7 +832,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
if root := getChtRoot(pm.chainDb, req.ChtNum); root != (common.Hash{}) {
if tr, _ := trie.New(root, pm.chainDb); tr != nil {
if tr, _ := trie.New(root, pm.chainDb, 0); tr != nil {
var encNumber [8]byte
binary.BigEndian.PutUint64(encNumber[:], req.BlockNum)
proof := tr.Prove(encNumber[:])

4
les/server.go
View file

@ -362,13 +362,13 @@ func makeCht(db ethdb.Database) bool {
var t *trie.Trie
if lastChtNum > 0 {
var err error
t, err = trie.New(getChtRoot(db, lastChtNum), db)
t, err = trie.New(getChtRoot(db, lastChtNum), db, 0)
if err != nil {
lastChtNum = 0
}
}
if lastChtNum == 0 {
t, _ = trie.New(common.Hash{}, db)
t, _ = trie.New(common.Hash{}, db, 0)
}
for num := lastChtNum * light.ChtFrequency; num < (lastChtNum+1)*light.ChtFrequency; num++ {

38
light/trie.go
View file

@ -24,7 +24,7 @@ import (
// LightTrie is an ODR-capable wrapper around trie.SecureTrie
type LightTrie struct {
trie *trie.SecureTrie
data *trie.SecureTrie
id *TrieID
odr OdrBackend
db ethdb.Database
@ -73,15 +73,25 @@ func (t *LightTrie) do(ctx context.Context, fallbackKey []byte, fn func() error)
return err
}
func (t *LightTrie) getMap() (ret *trie.SecureTrie, err error) {
if t.data == nil {
var tr trie.PersistentMap
tr, err = trie.New(t.id.Root, t.db, 0)
if err != nil {
return nil, err
}
t.data = trie.NewSecure(tr, t.db)
}
return t.data, nil
}
// Get returns the value for key stored in the trie.
// The value bytes must not be modified by the caller.
func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error) {
err = t.do(ctx, key, func() (err error) {
if t.trie == nil {
t.trie, err = trie.NewSecure(t.id.Root, t.db, 0)
}
if err == nil {
res, err = t.trie.TryGet(key)
var st *trie.SecureTrie
if st, err = t.getMap(); err == nil {
res, err = st.TryGet(key)
}
return
})
@ -96,11 +106,9 @@ func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error)
// stored in the trie.
func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
err = t.do(ctx, key, func() (err error) {
if t.trie == nil {
t.trie, err = trie.NewSecure(t.id.Root, t.db, 0)
}
if err == nil {
err = t.trie.TryUpdate(key, value)
var st *trie.SecureTrie
if st, err = t.getMap(); err == nil {
err = st.TryUpdate(key, value)
}
return
})
@ -110,11 +118,9 @@ func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) {
// Delete removes any existing value for key from the trie.
func (t *LightTrie) Delete(ctx context.Context, key []byte) (err error) {
err = t.do(ctx, key, func() (err error) {
if t.trie == nil {
t.trie, err = trie.NewSecure(t.id.Root, t.db, 0)
}
if err == nil {
err = t.trie.TryDelete(key)
var st *trie.SecureTrie
if st, err = t.getMap(); err == nil {
err = st.TryDelete(key)
}
return
})

2
miner/worker.go
View file

@ -617,7 +617,7 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB
continue
}
// Start executing the transaction
env.state.StartRecord(tx.Hash(), common.Hash{}, env.tcount)
env.state.SetTxContext(common.Hash{}, env.header.Number.Uint64(), tx.Hash(), env.tcount, bc)
err, logs := env.commitTransaction(tx, bc, gp)
switch {

180
trie/directcache.go Normal file
View file

@ -0,0 +1,180 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
)
var directCacheWrites = metrics.NewCounter("directcache/writes")
var directCacheHitTimer = metrics.NewTimer("directcache/timer/hits")
var directCacheMissTimer = metrics.NewTimer("directcache/timer/misses")
type cachedValue struct {
Value []byte
BlockNum uint64
BlockHash common.Hash
}
// CacheValidator can check whether a certain block is in the current canonical chain.
type CacheValidator interface {
IsCanonChainBlock(uint64, common.Hash) bool
}
type DirectCache struct {
data PersistentMap
db Database
keyPrefix []byte
blockNum uint64
blockHash common.Hash
validator CacheValidator
complete bool
dirty map[string]bool
}
type NullCacheValidator struct {}
func (cv *NullCacheValidator) IsCanonChainBlock(num uint64, hash common.Hash) bool {
return false
}
func NewDirectCache(pm PersistentMap, db Database, keyPrefix []byte, validator CacheValidator, complete bool) *DirectCache {
return &DirectCache{
data: pm,
db: db,
keyPrefix: keyPrefix,
validator: validator,
complete: complete,
dirty: make(map[string]bool),
}
}
func (dc *DirectCache) Iterator() *Iterator {
// Todo: If complete is true, implement an iterator over the DB instead.
return dc.data.Iterator()
}
func (dc *DirectCache) makeKey(key []byte) []byte {
return append(dc.keyPrefix, key...)
}
func (dc *DirectCache) Get(key []byte) []byte {
res, err := dc.TryGet(key)
if err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled error: %v", err)
}
return res
}
func (dc *DirectCache) TryGet(key []byte) ([]byte, error) {
start := time.Now()
dirty := dc.dirty[string(key)]
// Use the underlying object for dirty keys
if !dirty {
cacheKey := dc.makeKey(key)
if cached, ok := dc.getCached(cacheKey); ok {
directCacheHitTimer.UpdateSince(start)
return cached, nil
}
}
value, err := dc.data.TryGet(key)
if err != nil {
return value, err
}
if !dc.dirty[string(key)] {
// Flag the key as dirty so it gets written at commit time
dc.dirty[string(key)] = true
}
// Don't count fetches of dirty data as cache misses
if !dirty {
directCacheMissTimer.UpdateSince(start)
}
return value, nil
}
func (dc *DirectCache) getCached(key []byte) ([]byte, bool) {
enc, _ := dc.db.Get(key)
if len(enc) == 0 {
return nil, dc.complete
}
var data cachedValue
if err := rlp.DecodeBytes(enc, &data); err != nil && glog.V(logger.Error) {
glog.Errorf("Can't decode cached object at %x: %v", key, err)
return nil, false
}
canonical := dc.validator.IsCanonChainBlock(data.BlockNum, data.BlockHash)
return data.Value, canonical
}
func (dc *DirectCache) Update(key, value []byte) {
if err := dc.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled error: %v", err)
}
}
func (dc *DirectCache) TryUpdate(key, value []byte) error {
dc.dirty[string(key)] = true
return dc.data.TryUpdate(key, value)
}
func (dc *DirectCache) Delete(key []byte) {
if err := dc.TryDelete(key); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled error: %v", err)
}
}
func (dc *DirectCache) TryDelete(key []byte) error {
dc.dirty[string(key)] = true
return dc.data.TryDelete(key)
}
func (dc *DirectCache) CommitTo(dbw DatabaseWriter) (root common.Hash, err error) {
directCacheWrites.Inc(int64(len(dc.dirty)))
for k, _ := range dc.dirty {
v, err := dc.data.TryGet([]byte(k))
if err, ok := err.(*MissingNodeError); err != nil && !ok {
return common.Hash{}, err
}
if err := dc.putCache(dbw, []byte(k), v); err != nil {
return common.Hash{}, err
}
}
dc.dirty = make(map[string]bool)
return dc.data.CommitTo(dbw)
}
func (dc *DirectCache) putCache(dbw DatabaseWriter, key, value []byte) error {
enc, _ := rlp.EncodeToBytes(cachedValue{value, dc.blockNum, dc.blockHash})
if err := dbw.Put(append(dc.keyPrefix, key...), enc); err != nil {
return err
}
return nil
}

4
trie/proof_test.go
View file

@ -50,7 +50,7 @@ func TestProof(t *testing.T) {
}
func TestOneElementProof(t *testing.T) {
trie := new(Trie)
trie, _ := New(common.Hash{}, nil, 0)
updateString(trie, "k", "v")
proof := trie.Prove([]byte("k"))
if proof == nil {
@ -130,7 +130,7 @@ func BenchmarkVerifyProof(b *testing.B) {
}
func randomTrie(n int) (*Trie, map[string]*kv) {
trie := new(Trie)
trie, _ := New(common.Hash{}, nil, 0)
vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}

113
trie/secure_trie.go
View file

@ -26,6 +26,17 @@ var secureKeyPrefix = []byte("secure-key-")
const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
type PersistentMap interface {
Iterator() *Iterator
Get(key []byte) []byte
TryGet(key []byte) ([]byte, error)
Update(key, value []byte)
TryUpdate(key, value []byte) error
Delete(key []byte)
TryDelete(key []byte) error
CommitTo(db DatabaseWriter) (root common.Hash, err error)
}
// SecureTrie wraps a trie with key hashing. In a secure trie, all
// access operations hash the key using keccak256. This prevents
// calling code from creating long chains of nodes that
@ -37,75 +48,65 @@ const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
//
// SecureTrie is not safe for concurrent use.
type SecureTrie struct {
trie Trie
data PersistentMap
db Database
hashKeyBuf [secureKeyLength]byte
secKeyBuf [200]byte
secKeyCache map[string][]byte
secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
}
// NewSecure creates a trie with an existing root node from db.
//
// If root is the zero hash or the sha3 hash of an empty string, the
// trie is initially empty. Otherwise, New will panic if db is nil
// and returns MissingNodeError if the root node cannot be found.
//
// Accessing the trie loads nodes from db on demand.
// Loaded nodes are kept around until their 'cache generation' expires.
// A new cache generation is created by each call to Commit.
// cachelimit sets the number of past cache generations to keep.
func NewSecure(root common.Hash, db Database, cachelimit uint16) (*SecureTrie, error) {
// NewSecure creates a secure persistent map from an existing map.
func NewSecure(pm PersistentMap, db Database) *SecureTrie {
if pm == nil {
panic("NewSecure called with nil persistent map")
}
if db == nil {
panic("NewSecure called with nil database")
}
trie, err := New(root, db)
if err != nil {
return nil, err
}
trie.SetCacheLimit(cachelimit)
return &SecureTrie{trie: *trie}, nil
return &SecureTrie{data: pm, db: db}
}
// Get returns the value for key stored in the trie.
// Get returns the value for key stored in the map.
// The value bytes must not be modified by the caller.
func (t *SecureTrie) Get(key []byte) []byte {
res, err := t.TryGet(key)
if err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled trie error: %v", err)
glog.Errorf("Unhandled persistent map error: %v", err)
}
return res
}
// TryGet returns the value for key stored in the trie.
// TryGet returns the value for key stored in the map.
// The value bytes must not be modified by the caller.
// If a node was not found in the database, a MissingNodeError is returned.
func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
return t.trie.TryGet(t.hashKey(key))
return t.data.TryGet(t.hashKey(key))
}
// Update associates key with value in the trie. Subsequent calls to
// Update associates key with value in the map. Subsequent calls to
// Get will return value. If value has length zero, any existing value
// is deleted from the trie and calls to Get will return nil.
// is deleted from the map and calls to Get will return nil.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
// stored in the map.
func (t *SecureTrie) Update(key, value []byte) {
if err := t.TryUpdate(key, value); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled trie error: %v", err)
glog.Errorf("Unhandled persistent map error: %v", err)
}
}
// TryUpdate associates key with value in the trie. Subsequent calls to
// TryUpdate associates key with value in the map. Subsequent calls to
// Get will return value. If value has length zero, any existing value
// is deleted from the trie and calls to Get will return nil.
// is deleted from the map and calls to Get will return nil.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
// stored in the map.
//
// If a node was not found in the database, a MissingNodeError is returned.
func (t *SecureTrie) TryUpdate(key, value []byte) error {
hk := t.hashKey(key)
err := t.trie.TryUpdate(hk, value)
err := t.data.TryUpdate(hk, value)
if err != nil {
return err
}
@ -113,19 +114,19 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error {
return nil
}
// Delete removes any existing value for key from the trie.
// Delete removes any existing value for key from the map.
func (t *SecureTrie) Delete(key []byte) {
if err := t.TryDelete(key); err != nil && glog.V(logger.Error) {
glog.Errorf("Unhandled trie error: %v", err)
glog.Errorf("Unhandled persistent map error: %v", err)
}
}
// TryDelete removes any existing value for key from the trie.
// TryDelete removes any existing value for key from the map.
// If a node was not found in the database, a MissingNodeError is returned.
func (t *SecureTrie) TryDelete(key []byte) error {
hk := t.hashKey(key)
delete(t.getSecKeyCache(), string(hk))
return t.trie.TryDelete(hk)
return t.data.TryDelete(hk)
}
// GetKey returns the sha3 preimage of a hashed key that was
@ -134,51 +135,37 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
key, _ := t.trie.db.Get(t.secKey(shaKey))
key, _ := t.db.Get(t.secKey(shaKey))
return key
}
// Commit writes all nodes and the secure hash pre-images to the trie's database.
// Nodes are stored with their sha3 hash as the key.
//
// Committing flushes nodes from memory. Subsequent Get calls will load nodes
// from the database.
func (t *SecureTrie) Commit() (root common.Hash, err error) {
return t.CommitTo(t.trie.db)
}
func (t *SecureTrie) Hash() common.Hash {
return t.trie.Hash()
}
func (t *SecureTrie) Root() []byte {
return t.trie.Root()
}
func (t *SecureTrie) Iterator() *Iterator {
return t.trie.Iterator()
}
func (t *SecureTrie) NodeIterator() *NodeIterator {
return NewNodeIterator(&t.trie)
return t.data.Iterator()
}
// CommitTo writes all nodes and the secure hash pre-images to the given database.
// Nodes are stored with their sha3 hash as the key.
//
// Committing flushes nodes from memory. Subsequent Get calls will load nodes from
// the trie's database. Calling code must ensure that the changes made to db are
// written back to the trie's attached database before using the trie.
// the database. Calling code must ensure that the changes made to db are
// written back to the attached database before using the map.
func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
if err := t.CommitPreimages(); err != nil {
return common.Hash{}, err
}
return t.data.CommitTo(db)
}
func (t *SecureTrie) CommitPreimages() error {
if len(t.getSecKeyCache()) > 0 {
for hk, key := range t.secKeyCache {
if err := db.Put(t.secKey([]byte(hk)), key); err != nil {
return common.Hash{}, err
if err := t.db.Put(t.secKey([]byte(hk)), key); err != nil {
return err
}
}
t.secKeyCache = make(map[string][]byte)
}
return t.trie.CommitTo(db)
return nil
}
// secKey returns the database key for the preimage of key, as an ephemeral buffer.
@ -203,7 +190,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte {
}
// getSecKeyCache returns the current secure key cache, creating a new one if
// ownership changed (i.e. the current secure trie is a copy of another owning
// ownership changed (i.e. the current secure map is a copy of another owning
// the actual cache).
func (t *SecureTrie) getSecKeyCache() map[string][]byte {
if t != t.secKeyCacheOwner {

30
trie/secure_trie_test.go
View file

@ -27,17 +27,19 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
)
func newEmptySecure() *SecureTrie {
func newEmptySecure() (*Trie, *SecureTrie) {
db, _ := ethdb.NewMemDatabase()
trie, _ := NewSecure(common.Hash{}, db, 0)
return trie
tr, _ := New(common.Hash{}, db, 0)
st := NewSecure(tr, db)
return tr, st
}
// makeTestSecureTrie creates a large enough secure trie for testing.
func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
// Create an empty trie
db, _ := ethdb.NewMemDatabase()
trie, _ := NewSecure(common.Hash{}, db, 0)
tr, _ := New(common.Hash{}, db, 0)
trie := NewSecure(tr, db)
// Fill it with some arbitrary data
content := make(map[string][]byte)
@ -58,14 +60,14 @@ func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
trie.Update(key, val)
}
}
trie.Commit()
trie.CommitTo(db)
// Return the generated trie
return db, trie, content
}
func TestSecureDelete(t *testing.T) {
trie := newEmptySecure()
trie, st := newEmptySecure()
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@ -78,9 +80,9 @@ func TestSecureDelete(t *testing.T) {
}
for _, val := range vals {
if val.v != "" {
trie.Update([]byte(val.k), []byte(val.v))
st.Update([]byte(val.k), []byte(val.v))
} else {
trie.Delete([]byte(val.k))
st.Delete([]byte(val.k))
}
}
hash := trie.Hash()
@ -91,24 +93,24 @@ func TestSecureDelete(t *testing.T) {
}
func TestSecureGetKey(t *testing.T) {
trie := newEmptySecure()
trie.Update([]byte("foo"), []byte("bar"))
_, st := newEmptySecure()
st.Update([]byte("foo"), []byte("bar"))
key := []byte("foo")
value := []byte("bar")
seckey := crypto.Keccak256(key)
if !bytes.Equal(trie.Get(key), value) {
if !bytes.Equal(st.Get(key), value) {
t.Errorf("Get did not return bar")
}
if k := trie.GetKey(seckey); !bytes.Equal(k, key) {
if k := st.GetKey(seckey); !bytes.Equal(k, key) {
t.Errorf("GetKey returned %q, want %q", k, key)
}
}
func TestSecureTrieConcurrency(t *testing.T) {
// Create an initial trie and copy if for concurrent access
_, trie, _ := makeTestSecureTrie()
db, trie, _ := makeTestSecureTrie()
threads := runtime.NumCPU()
tries := make([]*SecureTrie, threads)
@ -137,7 +139,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
tries[index].Update(key, val)
}
}
tries[index].Commit()
tries[index].CommitTo(db)
}(i)
}
// Wait for all threads to finish

10
trie/sync_test.go
View file

@ -28,7 +28,7 @@ import (
func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
// Create an empty trie
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db)
trie, _ := New(common.Hash{}, db, 0)
// Fill it with some arbitrary data
content := make(map[string][]byte)
@ -59,7 +59,7 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
// content map.
func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
// Check root availability and trie contents
trie, err := New(common.BytesToHash(root), db)
trie, err := New(common.BytesToHash(root), db, 0)
if err != nil {
t.Fatalf("failed to create trie at %x: %v", root, err)
}
@ -76,7 +76,7 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
// checkTrieConsistency checks that all nodes in a trie are indeed present.
func checkTrieConsistency(db Database, root common.Hash) error {
// Create and iterate a trie rooted in a subnode
trie, err := New(root, db)
trie, err := New(root, db, 0)
if err != nil {
return nil // // Consider a non existent state consistent
}
@ -88,8 +88,8 @@ func checkTrieConsistency(db Database, root common.Hash) error {
// Tests that an empty trie is not scheduled for syncing.
func TestEmptyTrieSync(t *testing.T) {
emptyA, _ := New(common.Hash{}, nil)
emptyB, _ := New(emptyRoot, nil)
emptyA, _ := New(common.Hash{}, nil, 0)
emptyB, _ := New(emptyRoot, nil, 0)
for i, trie := range []*Trie{emptyA, emptyB} {
db, _ := ethdb.NewMemDatabase()

21
trie/trie.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package trie implements Merkle Patricia Tries.
// Package trie implements Merkle Patricia tries.
package trie
import (
@ -73,11 +73,11 @@ type DatabaseWriter interface {
Put(key, value []byte) error
}
// Trie is a Merkle Patricia Trie.
// trie is a Merkle Patricia trie.
// The zero value is an empty trie with no database.
// Use New to create a trie that sits on top of a database.
//
// Trie is not safe for concurrent use.
// trie is not safe for concurrent use.
type Trie struct {
root node
db Database
@ -90,12 +90,6 @@ type Trie struct {
cachegen, cachelimit uint16
}
// SetCacheLimit sets the number of 'cache generations' to keep.
// A cache generations is created by a call to Commit.
func (t *Trie) SetCacheLimit(l uint16) {
t.cachelimit = l
}
// newFlag returns the cache flag value for a newly created node.
func (t *Trie) newFlag() nodeFlag {
return nodeFlag{dirty: true, gen: t.cachegen}
@ -107,11 +101,14 @@ func (t *Trie) newFlag() nodeFlag {
// trie is initially empty and does not require a database. Otherwise,
// New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand.
func New(root common.Hash, db Database) (*Trie, error) {
trie := &Trie{db: db, originalRoot: root}
//
// cacheLimit is the number of 'cache generations' to keep.
// A cache generations is created by a call to Commit.
func New(root common.Hash, db Database, cacheLimit uint16) (*Trie, error) {
trie := &Trie{db: db, originalRoot: root, cachelimit: cacheLimit}
if (root != common.Hash{}) && root != emptyRoot {
if db == nil {
panic("trie.New: cannot use existing root without a database")
panic("Trie.New: cannot use existing root without a database")
}
rootnode, err := trie.resolveHash(root[:], nil, nil)
if err != nil {

41
trie/trie_test.go
View file

@ -40,7 +40,7 @@ func init() {
// Used for testing
func newEmpty() *Trie {
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db)
trie, _ := New(common.Hash{}, db, 0)
return trie
}
@ -63,7 +63,7 @@ func TestNull(t *testing.T) {
func TestMissingRoot(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db)
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), db, 0)
if trie != nil {
t.Error("New returned non-nil trie for invalid root")
}
@ -74,36 +74,36 @@ func TestMissingRoot(t *testing.T) {
func TestMissingNode(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
trie, _ := New(common.Hash{}, db)
trie, _ := New(common.Hash{}, db, 0)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _ := trie.Commit()
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err := trie.TryGet([]byte("120000"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err = trie.TryGet([]byte("120099"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
err = trie.TryDelete([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
@ -111,31 +111,31 @@ func TestMissingNode(t *testing.T) {
db.Delete(common.FromHex("e1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9"))
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err = trie.TryGet([]byte("120000"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err = trie.TryGet([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
_, err = trie.TryGet([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
err = trie.TryUpdate([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(root, db)
trie, _ = New(root, db, 0)
err = trie.TryDelete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
@ -263,7 +263,7 @@ func TestReplication(t *testing.T) {
}
// create a new trie on top of the database and check that lookups work.
trie2, err := New(exp, trie.db)
trie2, err := New(exp, trie.db, 0)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
@ -332,8 +332,7 @@ func TestCacheUnload(t *testing.T) {
// The branch containing it is loaded from DB exactly two times:
// in the 0th and 6th iteration.
db := &countingDB{Database: trie.db, gets: make(map[string]int)}
trie, _ = New(root, db)
trie.SetCacheLimit(5)
trie, _ = New(root, db, 5)
for i := 0; i < 12; i++ {
getString(trie, key1)
trie.Commit()
@ -417,7 +416,7 @@ func randRead(r *rand.Rand, b []byte) {
func runRandTest(rt randTest) bool {
db, _ := ethdb.NewMemDatabase()
tr, _ := New(common.Hash{}, db)
tr, _ := New(common.Hash{}, db, 0)
values := make(map[string]string) // tracks content of the trie
for _, step := range rt {
@ -446,13 +445,13 @@ func runRandTest(rt randTest) bool {
if err != nil {
panic(err)
}
newtr, err := New(hash, db)
newtr, err := New(hash, db, 0)
if err != nil {
panic(err)
}
tr = newtr
case opItercheckhash:
checktr, _ := New(common.Hash{}, nil)
checktr, _ := New(common.Hash{}, nil, 0)
it := tr.Iterator()
for it.Next() {
checktr.Update(it.Key, it.Value)
@ -520,10 +519,10 @@ func BenchmarkHashLE(b *testing.B) { benchHash(b, binary.LittleEndian) }
const benchElemCount = 20000
func benchGet(b *testing.B, commit bool) {
trie := new(Trie)
trie, _ := New(common.Hash{}, nil, 0)
if commit {
_, tmpdb := tempDB()
trie, _ = New(common.Hash{}, tmpdb)
trie, _ = New(common.Hash{}, tmpdb, 0)
}
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {