core/state/snapshot: hack in memory cache into teh disk layer

core/state/snapshot/difflayer.go

@@ -254,6 +254,8 @@ func (dl *diffLayer) Cap(layers int, memory uint64) (uint64, uint64) {
 	// Push all the accounts into the database
 	for hash, data := range parent.accountData {
 		rawdb.WriteAccountSnapshot(batch, hash, data)
+		base.cache.Set(string(hash[:]), data)
+
 		if batch.ValueSize() > ethdb.IdealBatchSize {
 			if err := batch.Write(); err != nil {
 				log.Crit("Failed to write account snapshot", "err", err)
@@ -265,6 +267,7 @@ func (dl *diffLayer) Cap(layers int, memory uint64) (uint64, uint64) {
 	for accountHash, storage := range parent.storageData {
 		for storageHash, data := range storage {
 			rawdb.WriteStorageSnapshot(batch, accountHash, storageHash, data)
+			base.cache.Set(string(append(accountHash[:], storageHash[:]...)), data)
 		}
 		if batch.ValueSize() > ethdb.IdealBatchSize {
 			if err := batch.Write(); err != nil {

core/state/snapshot/disklayer.go

@@ -17,6 +17,7 @@
 package snapshot
 
 import (
+	"github.com/allegro/bigcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -27,6 +28,7 @@ import (
 type diskLayer struct {
 	journal string              // Path of the snapshot journal to use on shutdown
 	db      ethdb.KeyValueStore // Key-value store containing the base snapshot
+	cache   *bigcache.BigCache  // Cache to avoid hitting the disk for direct access
 
 	number uint64      // Block number of the base snapshot
 	root   common.Hash // Root hash of the base snapshot
@@ -54,13 +56,43 @@ func (dl *diskLayer) Account(hash common.Hash) *Account {
 // AccountRLP directly retrieves the account RLP associated with a particular
 // hash in the snapshot slim data format.
 func (dl *diskLayer) AccountRLP(hash common.Hash) []byte {
-	return rawdb.ReadAccountSnapshot(dl.db, hash)
+	key := string(hash[:])
+
+	// Try to retrieve the account from the memory cache
+	if blob, err := dl.cache.Get(key); err == nil {
+		snapshotCleanHitMeter.Mark(1)
+		snapshotCleanReadMeter.Mark(int64(len(blob)))
+		return blob
+	}
+	// Cache doesn't contain account, pull from disk and cache for later
+	blob := rawdb.ReadAccountSnapshot(dl.db, hash)
+	dl.cache.Set(key, blob)
+
+	snapshotCleanMissMeter.Mark(1)
+	snapshotCleanWriteMeter.Mark(int64(len(blob)))
+
+	return blob
 }
 
 // Storage directly retrieves the storage data associated with a particular hash,
 // within a particular account.
 func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) []byte {
-	return rawdb.ReadStorageSnapshot(dl.db, accountHash, storageHash)
+	key := string(append(accountHash[:], storageHash[:]...))
+
+	// Try to retrieve the storage slot from the memory cache
+	if blob, err := dl.cache.Get(key); err == nil {
+		snapshotCleanHitMeter.Mark(1)
+		snapshotCleanReadMeter.Mark(int64(len(blob)))
+		return blob
+	}
+	// Cache doesn't contain storage slot, pull from disk and cache for later
+	blob := rawdb.ReadStorageSnapshot(dl.db, accountHash, storageHash)
+	dl.cache.Set(key, blob)
+
+	snapshotCleanMissMeter.Mark(1)
+	snapshotCleanWriteMeter.Mark(int64(len(blob)))
+
+	return blob
 }
 
 // Update creates a new layer on top of the existing snapshot diff tree with

core/state/snapshot/generate.go

@@ -22,6 +22,7 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/allegro/bigcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -188,9 +189,17 @@ func generateSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64,
 		return nil, err
 	}
 	// New snapshot generated, construct a brand new base layer
+	cache, _ := bigcache.NewBigCache(bigcache.Config{ // TODO(karalabe): dedup
+		Shards:             1024,
+		LifeWindow:         time.Hour,
+		MaxEntriesInWindow: 512 * 1024,
+		MaxEntrySize:       512,
+		HardMaxCacheSize:   512,
+	})
 	return &diskLayer{
 		journal: journal,
 		db:      db,
+		cache:   cache,
 		number:  headNumber,
 		root:    headRoot,
 	}, nil

core/state/snapshot/snapshot.go

@@ -22,11 +22,21 @@ import (
 	"fmt"
 	"os"
 	"sync"
+	"time"
 
+	"github.com/allegro/bigcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+	snapshotCleanHitMeter   = metrics.NewRegisteredMeter("state/snapshot/clean/hit", nil)
+	snapshotCleanMissMeter  = metrics.NewRegisteredMeter("state/snapshot/clean/miss", nil)
+	snapshotCleanReadMeter  = metrics.NewRegisteredMeter("state/snapshot/clean/read", nil)
+	snapshotCleanWriteMeter = metrics.NewRegisteredMeter("state/snapshot/clean/write", nil)
 )
 
 // Snapshot represents the functionality supported by a snapshot storage layer.
@@ -190,9 +200,17 @@ func loadSnapshot(db ethdb.KeyValueStore, journal string, headNumber uint64, hea
 	if root == (common.Hash{}) {
 		return nil, errors.New("missing or corrupted snapshot")
 	}
+	cache, _ := bigcache.NewBigCache(bigcache.Config{ // TODO(karalabe): dedup
+		Shards:             1024,
+		LifeWindow:         time.Hour,
+		MaxEntriesInWindow: 512 * 1024,
+		MaxEntrySize:       512,
+		HardMaxCacheSize:   512,
+	})
 	base := &diskLayer{
 		journal: journal,
 		db:      db,
+		cache:   cache,
 		number:  number,
 		root:    root,
 	}