forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 22:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

cmd, core/state, eth, trie: implement memcache for the direct cache

Browse source
This commit is contained in:
Péter Szilágyi 2016-11-02 13:36:13 +02:00
parent f40e3442a1
commit f8be5d9367
No known key found for this signature in database
GPG key ID: 119A76381CCB7DD2
4 changed files with 95 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/geth/chaincmd.go
View file

@ -136,9 +136,11 @@ func importChain(ctx *cli.Context) error {
fmt.Println(stats)
fmt.Printf("Trie cache misses: %d\n", trie.CacheMisses())
fmt.Printf("Trie cache unloads: %d\n", trie.CacheUnloads())
fmt.Printf("Direct cache reads: %d\n", trie.DirectCacheReads())
fmt.Printf("Direct cache writes: %d\n", trie.DirectCacheWrites())
fmt.Printf("Direct cache misses: %d\n\n", trie.DirectCacheMisses())
fmt.Printf("Direct memory cache hits: %d\n", trie.DirectCacheMemcacheHits())
fmt.Printf("Direct memory cache misses: %d\n", trie.DirectCacheMemcacheMisses())
fmt.Printf("Direct database cache reads: %d\n", trie.DirectCacheReads())
fmt.Printf("Direct database cache writes: %d\n", trie.DirectCacheWrites())
fmt.Printf("Direct database cache misses: %d\n\n", trie.DirectCacheMisses())
// Print the memory statistics used by the importing
mem := new(runtime.MemStats)

9
core/state/statedb.go
View file

@ -46,6 +46,9 @@ const (
// Number of codehash->size associations to keep.
codeSizeCacheSize = 100000
// Number of recent accessed accounts to keep in memory
accountsMemoryCacheSize = 4096
)
type revision struct {
@ -65,6 +68,7 @@ type StateDB struct {
cacheComplete bool // True if we know that the directcache is complete
pastTries []*trie.Trie
codeSizeCache *lru.Cache
accountsMemCache *lru.ARCCache
// This map holds 'live' objects, which will get modified while processing a state transition.
stateObjects map[common.Address]*StateObject
@ -95,10 +99,12 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
}
csc, _ := lru.New(codeSizeCacheSize)
amc, _ := lru.NewARC(accountsMemoryCacheSize)
ret := &StateDB{
db: db,
trie: tr,
codeSizeCache: csc,
accountsMemCache: amc,
stateObjects: make(map[common.Address]*StateObject),
stateObjectsDirty: make(map[common.Address]struct{}),
refund: new(big.Int),
@ -122,6 +128,7 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) {
db: self.db,
trie: tr,
codeSizeCache: self.codeSizeCache,
accountsMemCache: self.accountsMemCache,
stateObjects: make(map[common.Address]*StateObject),
stateObjectsDirty: make(map[common.Address]struct{}),
refund: new(big.Int),
@ -193,7 +200,7 @@ func (self *StateDB) SetBlockContext(blockHash common.Hash, blockNum uint64, val
}
}
storage := trie.NewDirectCache(self.trie, self.db, DirectCachePrefix, blockNum, blockHash, validator, self.cacheComplete)
storage := trie.NewDirectCache(self.trie, self.db, self.accountsMemCache, DirectCachePrefix, blockNum, blockHash, validator, self.cacheComplete)
self.storage = trie.NewSecure(storage, self.db)
}

2
eth/db_upgrade.go
View file

@ -371,7 +371,7 @@ func populateDirectCache(db ethdb.Database, bc *core.BlockChain) error {
if err != nil {
return err
}
dc := trie.NewDirectCache(tr, db, state.DirectCachePrefix, blockNum, blockHash, bc, false)
dc := trie.NewDirectCache(tr, db, nil, state.DirectCachePrefix, blockNum, blockHash, bc, false)
go func() {
for core.GetBlockNumber(db, core.GetHeadBlockHash(db)) < blockNum+8 {
time.Sleep(10 * time.Second)

79
trie/directcache.go
View file

@ -17,16 +17,17 @@
package trie
import (
"fmt"
"errors"
"time"
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
lru "github.com/hashicorp/golang-lru"
)
var (
@ -35,7 +36,10 @@ var (
directCacheWrites = metrics.NewCounter("directcache/writes")
directCacheHits = metrics.NewCounter("directcache/hits")
directCacheMisses = metrics.NewCounter("directcache/misses")
directCacheMemHits = metrics.NewCounter("directcache/memcache/hits")
directCacheMemMisses = metrics.NewCounter("directcache/memcache/misses")
directCacheTimer = metrics.NewTimer("directcache/timer")
NotFound = errors.New("Cache entry not found")
MigrationPrefix = []byte("directstatecachemigration:")
)
@ -56,6 +60,8 @@ type CacheValidator interface {
type DirectCache struct {
data PersistentMap
db Database
memcache *lru.ARCCache
keyPrefix []byte
blockNum uint64
blockHash common.Hash
@ -70,10 +76,14 @@ func (cv *NullCacheValidator) IsCanonChainBlock(num uint64, hash common.Hash) bo
return false
}
func NewDirectCache(pm PersistentMap, db Database, keyPrefix []byte, blockNum uint64, blockHash common.Hash, validator CacheValidator, complete bool) *DirectCache {
func NewDirectCache(pm PersistentMap, db Database, memcache *lru.ARCCache, keyPrefix []byte, blockNum uint64, blockHash common.Hash, validator CacheValidator, complete bool) *DirectCache {
if memcache == nil {
memcache, _ = lru.NewARC(0)
}
return &DirectCache{
data: pm,
db: db,
memcache: memcache,
keyPrefix: keyPrefix,
blockNum: blockNum,
blockHash: blockHash,
@ -125,17 +135,28 @@ func (dc *DirectCache) TryGet(key []byte) ([]byte, error) {
}
func (dc *DirectCache) getCached(key []byte) ([]byte, bool) {
data, blockNum, blockHash, err := GetDirectCache(dc.keyPrefix, key, dc.db)
if err != nil {
// Try to retrieve the object from the memcache
var entry *cachedValue
if cached, ok := dc.memcache.Get(string(key)); ok {
directCacheMemHits.Inc(1)
entry = cached.(*cachedValue)
}
// If the memcache missed, reach down to the database
var err error
if entry == nil {
directCacheMemMisses.Inc(1)
if entry, err = getDirectCache(dc.keyPrefix, key, dc.db); err != nil {
if err == NotFound {
return nil, dc.complete
}
glog.Errorf("Error retrieving direct cache data: %v", err)
return nil, false
}
canonical := dc.blockNum > 0 && blockNum < dc.blockNum && dc.validator.IsCanonChainBlock(blockNum, blockHash)
return data, canonical
dc.memcache.Add(string(key), entry)
}
canonical := dc.blockNum > 0 && entry.BlockNum < dc.blockNum && dc.validator.IsCanonChainBlock(entry.BlockNum, entry.BlockHash)
return entry.Value, canonical
}
func (dc *DirectCache) Update(key, value []byte) {
@ -170,6 +191,7 @@ func (dc *DirectCache) CommitTo(dbw DatabaseWriter) (root common.Hash, err error
if err := WriteDirectCache(dc.keyPrefix, []byte(k), v, dc.blockNum, dc.blockHash, dbw); err != nil {
return err
}
dc.memcache.Add(k, &cachedValue{v, dc.blockNum, dc.blockHash})
}
return nil
}); err != nil {
@ -201,7 +223,7 @@ func (dc *DirectCache) Populate() (err error) {
}
i += 1
if i % 10000 == 0 && glog.V(logger.Info) {
if i%10000 == 0 && glog.V(logger.Info) {
glog.V(logger.Info).Infof("Constructing direct cache: processed %v entries, written %v", i, writes)
}
}
@ -214,7 +236,7 @@ type cachedValue struct {
BlockHash common.Hash
}
func DirectCacheTransaction(tx func() (error)) error {
func DirectCacheTransaction(tx func() error) error {
directCacheLock.Lock()
directCacheLocked = true
defer func() {
@ -241,22 +263,31 @@ func WriteDirectCache(prefix, key, value []byte, number uint64, hash common.Hash
return dbw.Put(append(prefix, key...), enc)
}
// getDirectCache retrieves a value node directly from the database along with
// block metadata to validate its relevancy.
func getDirectCache(prefix, key []byte, db Database) (*cachedValue, error) {
defer func(start time.Time) { directCacheTimer.UpdateSince(start) }(time.Now())
enc, _ := db.Get(append(prefix, key...))
if len(enc) == 0 {
return nil, NotFound
}
data := new(cachedValue)
if err := rlp.DecodeBytes(enc, data); err != nil {
return nil, fmt.Errorf("Can't decode cached object at %x: %v", key, err)
}
return data, nil
}
// GetDirectCache retrieves a value node directly from the database along with
// block metadata to validate its relevancy.
//
// The method is meant to be used by code that circumvents the state database
// and its integrated cache, namely during fast sync and database upgrades.
func GetDirectCache(prefix, key []byte, db Database) ([]byte, uint64, common.Hash, error) {
defer func(start time.Time) { directCacheTimer.UpdateSince(start) }(time.Now())
enc, _ := db.Get(append(prefix, key...))
if len(enc) == 0 {
return nil, 0, common.Hash{}, NotFound
}
var data cachedValue
if err := rlp.DecodeBytes(enc, &data); err != nil {
return nil, 0, common.Hash{}, fmt.Errorf("Can't decode cached object at %x: %v", key, err)
data, err := getDirectCache(prefix, key, db)
if err != nil {
return nil, 0, common.Hash{}, err
}
return data.Value, data.BlockNum, data.BlockHash, nil
}
@ -317,3 +348,11 @@ func DirectCacheWrites() int64 {
func DirectCacheMisses() int64 {
return directCacheMisses.Count()
}
func DirectCacheMemcacheHits() int64 {
return directCacheMemHits.Count()
}
func DirectCacheMemcacheMisses() int64 {
return directCacheMemMisses.Count()
}