feat: add snapshot multiversion cache

core/state/snapshot/difflayer.go

@@ -17,6 +17,7 @@
 package snapshot
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math"
@@ -28,6 +29,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	bloomfilter "github.com/holiman/bloomfilter/v2"
 )
@@ -103,6 +105,9 @@ type diffLayer struct {
 	parent snapshot   // Parent snapshot modified by this one, never nil
 	memory uint64     // Approximate guess as to how much memory we use
 
+	diffLayerID       uint64                     // diffLayerID is memory temp value, child_id = parent_id + 1, which is used as multi-version cache item version.
+	multiVersionCache *MultiVersionSnapshotCache // multiVersionCache is used to speed up the recursive query of 128 difflayers.
+
 	root  common.Hash // Root hash to which this snapshot diff belongs to
 	stale atomic.Bool // Signals that the layer became stale (state progressed)
 
@@ -154,8 +159,12 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s
 	}
 	switch parent := parent.(type) {
 	case *diskLayer:
+		dl.diffLayerID = 1
+		dl.multiVersionCache = NewMultiVersionSnapshotCache()
 		dl.rebloom(parent)
 	case *diffLayer:
+		dl.diffLayerID = parent.diffLayerID + 1
+		dl.multiVersionCache = parent.multiVersionCache
 		dl.rebloom(parent.origin)
 	default:
 		panic("unknown parent type")
@@ -183,6 +192,14 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s
 	return dl
 }
 
+func (dl *diffLayer) AddToCache() {
+	dl.multiVersionCache.Add(dl)
+}
+
+func (dl *diffLayer) RemoveFromCache() {
+	dl.multiVersionCache.Remove(dl)
+}
+
 // rebloom discards the layer's current bloom and rebuilds it from scratch based
 // on the parent's and the local diffs.
 func (dl *diffLayer) rebloom(origin *diskLayer) {
@@ -272,6 +289,46 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
 		dl.lock.RUnlock()
 		return nil, ErrSnapshotStale
 	}
+
+	{
+		// try fastpath
+		data, needTryDisk, err := dl.multiVersionCache.QueryAccount(dl.diffLayerID, dl.root, hash)
+		if err == nil {
+			if needTryDisk {
+				data, err = dl.origin.AccountRLP(hash)
+				diffMultiVersionCacheMissMeter.Mark(1)
+			} else {
+				diffMultiVersionCacheHitMeter.Mark(1)
+				diffMultiVersionCacheReadMeter.Mark(int64(len(data)))
+			}
+			if err != nil {
+				log.Warn("Account has bug due to query disklayer", "error", err)
+				diffMultiVersionCacheBugMeter.Mark(1)
+			}
+			dl.lock.RUnlock()
+
+			{
+				// todo: double check
+				expectedData, expectedErr := dl.accountRLP(hash, 0)
+				if !bytes.Equal(data, expectedData) {
+					log.Error("Has bug",
+						"query_version", dl.diffLayerID,
+						"query_root", dl.root,
+						"account_hash", hash,
+						"actual_hit_disk", needTryDisk,
+						"actual_disk_root", dl.origin.Root(),
+						"actual_data_len", len(data),
+						"expected_data_len", len(expectedData),
+						"actual_error", err,
+						"expected_error", expectedErr)
+				}
+			}
+			return data, err
+		}
+		log.Warn("Account has bug due to query multi version cache", "error", err)
+		diffMultiVersionCacheBugMeter.Mark(1)
+	}
+
 	// Check the bloom filter first whether there's even a point in reaching into
 	// all the maps in all the layers below
 	hit := dl.diffed.ContainsHash(accountBloomHash(hash))
@@ -345,6 +402,47 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
 		dl.lock.RUnlock()
 		return nil, ErrSnapshotStale
 	}
+
+	{
+		// try fastpath
+		data, needTryDisk, err := dl.multiVersionCache.QueryStorage(dl.diffLayerID, dl.root, accountHash, storageHash)
+		if err == nil {
+			if needTryDisk {
+				data, err = dl.origin.Storage(accountHash, storageHash)
+				diffMultiVersionCacheMissMeter.Mark(1)
+			} else {
+				diffMultiVersionCacheHitMeter.Mark(1)
+				diffMultiVersionCacheReadMeter.Mark(int64(len(data)))
+			}
+			if err != nil {
+				log.Warn("Storage has bug due to query disklayer", "error", err)
+				diffMultiVersionCacheBugMeter.Mark(1)
+			}
+			dl.lock.RUnlock()
+
+			{
+				// todo: double check
+				expectedData, expectedErr := dl.storage(accountHash, storageHash, 0)
+				if !bytes.Equal(data, expectedData) {
+					log.Warn("Has bug",
+						"query_version", dl.diffLayerID,
+						"query_root", dl.root,
+						"account_hash", accountHash,
+						"storage_hash", storageHash,
+						"actual_hit_disk", needTryDisk,
+						"actual_disk_root", dl.origin.Root(),
+						"actual_data_len", len(data),
+						"expected_data_len", len(expectedData),
+						"actual_error", err,
+						"expected_error", expectedErr)
+				}
+			}
+			return data, err
+		}
+		log.Warn("Storage has bug due to query multi version cache", "error", err)
+		diffMultiVersionCacheBugMeter.Mark(1)
+	}
+
 	hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash))
 	if !hit {
 		hit = dl.diffed.ContainsHash(destructBloomHash(accountHash))
@@ -462,6 +560,8 @@ func (dl *diffLayer) flatten() snapshot {
 	return &diffLayer{
 		parent:            parent.parent,
 		origin:            parent.origin,
+		diffLayerID:       dl.diffLayerID,
+		multiVersionCache: dl.multiVersionCache,
 		root:              dl.root,
 		destructSet:       parent.destructSet,
 		accountData:       parent.accountData,

core/state/snapshot/journal.go

@@ -110,7 +110,9 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou
 	// etc.), we just discard all diffs and try to recover them later.
 	var current snapshot = base
 	err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error {
-		current = newDiffLayer(current, root, destructSet, accountData, storageData)
+		diff := newDiffLayer(current, root, destructSet, accountData, storageData)
+		diff.AddToCache() // add to multi-version cache from the journal at startup.
+		current = diff
 		return nil
 	})
 	if err != nil {

core/state/snapshot/metrics.go

@@ -50,4 +50,11 @@ var (
 	snapStorageWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/write", nil)
 	// snapStorageCleanCounter measures time spent on deleting storages
 	snapStorageCleanCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/clean", nil)
+
+	// multi version cache metrics
+	diffMultiVersionCacheHitMeter    = metrics.NewRegisteredMeter("pathdb/difflayer/multiversioncache/hit", nil)
+	diffMultiVersionCacheReadMeter   = metrics.NewRegisteredMeter("pathdb/difflayer/multiversioncache/read", nil)
+	diffMultiVersionCacheMissMeter   = metrics.NewRegisteredMeter("pathdb/difflayer/multiversioncache/miss", nil)
+	diffMultiVersionCacheBugMeter    = metrics.NewRegisteredMeter("pathdb/difflayer/multiversioncache/bug", nil)
+	diffMultiVersionCacheLengthGauge = metrics.NewRegisteredGauge("pathdb/difflayer/multiversioncache/size", nil)
 )

core/state/snapshot/multiversion_cache.go

@@ -0,0 +1,541 @@
+package snapshot
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const (
+	batchGCNumber = 100
+)
+
+type multiVersionItem interface {
+	Version() uint64
+}
+
+type multiVersionItemSlice[T multiVersionItem] struct {
+	data []T
+}
+
+func newMultiVersionItemSlice[T multiVersionItem](inputData []T) *multiVersionItemSlice[T] {
+	c := multiVersionItemSlice[T]{}
+	c.data = inputData
+	return &c
+}
+
+// SortByVersion is used to resort the multi-version slice by version.
+func (a *multiVersionItemSlice[T]) SortByVersion() []T {
+	sort.Sort(a)
+	return a.data
+}
+
+func (a *multiVersionItemSlice[T]) Len() int {
+	return len(a.data)
+}
+
+func (a *multiVersionItemSlice[T]) Swap(i, j int) {
+	a.data[i], a.data[j] = a.data[j], a.data[i]
+}
+
+func (a *multiVersionItemSlice[T]) Less(i, j int) bool {
+	return a.data[j].Version() > a.data[i].Version()
+}
+
+type destructCacheItem struct {
+	version uint64
+	root    common.Hash
+}
+
+func (di *destructCacheItem) Version() uint64 {
+	return di.version
+}
+
+var _ multiVersionItem = &destructCacheItem{}
+
+type accountCacheItem struct {
+	version uint64
+	root    common.Hash
+	data    []byte
+}
+
+func (ai *accountCacheItem) Version() uint64 {
+	return ai.version
+}
+
+var _ multiVersionItem = &accountCacheItem{}
+
+type storageCacheItem struct {
+	version uint64
+	root    common.Hash
+	data    []byte
+}
+
+func (si *storageCacheItem) Version() uint64 {
+	return si.version
+}
+
+var _ multiVersionItem = &storageCacheItem{}
+
+func cloneParentMap(parentMap map[common.Hash]struct{}) map[common.Hash]struct{} {
+	cloneMap := make(map[common.Hash]struct{})
+	for k := range parentMap {
+		cloneMap[k] = struct{}{}
+	}
+	return cloneMap
+}
+
+type MultiVersionSnapshotCache struct {
+	lock             sync.RWMutex
+	destructCache    map[common.Hash][]*destructCacheItem
+	accountDataCache map[common.Hash][]*accountCacheItem
+	storageDataCache map[common.Hash]map[common.Hash][]*storageCacheItem
+	minVersion       uint64 // bottom version
+	diffLayerParent  map[common.Hash]map[common.Hash]struct{}
+	cacheItemNumber  int64
+
+	deltaRemoveQueue []*diffLayer
+}
+
+func NewMultiVersionSnapshotCache() *MultiVersionSnapshotCache {
+	c := &MultiVersionSnapshotCache{
+		destructCache:    make(map[common.Hash][]*destructCacheItem),
+		accountDataCache: make(map[common.Hash][]*accountCacheItem),
+		storageDataCache: make(map[common.Hash]map[common.Hash][]*storageCacheItem),
+		minVersion:       0,
+		diffLayerParent:  make(map[common.Hash]map[common.Hash]struct{}),
+		cacheItemNumber:  0,
+	}
+	go c.loopDelayGC()
+	return c
+}
+
+func (c *MultiVersionSnapshotCache) checkParent(childRoot common.Hash, parentRoot common.Hash) bool {
+	if c == nil {
+		return false
+	}
+	if _, exist := c.diffLayerParent[childRoot]; !exist {
+		return false
+	}
+	if _, exist := c.diffLayerParent[childRoot][parentRoot]; !exist {
+		return false
+	}
+	return true
+}
+
+func (c *MultiVersionSnapshotCache) Add(ly *diffLayer) {
+	if c == nil || ly == nil {
+		return
+	}
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	log.Info("Add difflayer to snapshot multiversion cache", "root", ly.root, "version_id", ly.diffLayerID, "current_cache_item_number", c.cacheItemNumber)
+
+	for hash := range ly.destructSet {
+		if multiVersionItems, exist := c.destructCache[hash]; exist {
+			multiVersionItems = append(multiVersionItems, &destructCacheItem{version: ly.diffLayerID, root: ly.root})
+			c.destructCache[hash] = multiVersionItems
+		} else {
+			c.destructCache[hash] = []*destructCacheItem{&destructCacheItem{version: ly.diffLayerID, root: ly.root}}
+		}
+		c.cacheItemNumber++
+		//log.Info("Add destruct to cache",
+		//	"cache_account_hash", hash,
+		//	"cache_version", ly.diffLayerID,
+		//	"cache_root", ly.root)
+	}
+	// sorted by version
+	for hash := range c.destructCache {
+		c.destructCache[hash] = newMultiVersionItemSlice[*destructCacheItem](c.destructCache[hash]).SortByVersion()
+	}
+
+	for hash, aData := range ly.accountData {
+		if multiVersionItems, exist := c.accountDataCache[hash]; exist {
+			multiVersionItems = append(multiVersionItems, &accountCacheItem{version: ly.diffLayerID, root: ly.root, data: aData})
+			c.accountDataCache[hash] = multiVersionItems
+		} else {
+			c.accountDataCache[hash] = []*accountCacheItem{&accountCacheItem{version: ly.diffLayerID, root: ly.root, data: aData}}
+		}
+		c.cacheItemNumber++
+		//log.Info("Add account to cache",
+		//	"cache_account_hash", hash,
+		//	"cache_version", ly.diffLayerID,
+		//	"cache_root", ly.root,
+		//	"cache_data_len", len(aData))
+	}
+
+	// sorted by version
+	for hash := range c.accountDataCache {
+		c.accountDataCache[hash] = newMultiVersionItemSlice[*accountCacheItem](c.accountDataCache[hash]).SortByVersion()
+	}
+
+	for accountHash, slots := range ly.storageData {
+		if _, exist := c.storageDataCache[accountHash]; !exist {
+			c.storageDataCache[accountHash] = make(map[common.Hash][]*storageCacheItem)
+		}
+		for storageHash, sData := range slots {
+			if multiVersionItems, exist := c.storageDataCache[accountHash][storageHash]; exist {
+				multiVersionItems = append(multiVersionItems, &storageCacheItem{version: ly.diffLayerID, root: ly.root, data: sData})
+				c.storageDataCache[accountHash][storageHash] = multiVersionItems
+			} else {
+				c.storageDataCache[accountHash][storageHash] = []*storageCacheItem{&storageCacheItem{version: ly.diffLayerID, root: ly.root, data: sData}}
+			}
+			c.cacheItemNumber++
+			//log.Info("Add storage to cache",
+			//	"cache_account_hash", accountHash,
+			//	"cache_storage_hash", storageHash,
+			//	"cache_version", ly.diffLayerID,
+			//	"cache_root", ly.root,
+			//	"cache_data_len", len(sData))
+		}
+	}
+	// sorted by version
+	for ahash := range c.storageDataCache {
+		for shash := range c.storageDataCache[ahash] {
+			c.storageDataCache[ahash][shash] = newMultiVersionItemSlice[*storageCacheItem](c.storageDataCache[ahash][shash]).SortByVersion()
+		}
+	}
+
+	if parentDiffLayer, ok := ly.parent.(*diffLayer); ok {
+		if parentLayerParent, exist := c.diffLayerParent[parentDiffLayer.root]; exist {
+			clonedParentLayerParent := cloneParentMap(parentLayerParent)
+			clonedParentLayerParent[ly.root] = struct{}{}
+			c.diffLayerParent[ly.root] = clonedParentLayerParent
+		} else {
+			log.Warn("Impossible branch, maybe there is a bug.")
+		}
+	} else {
+		c.diffLayerParent[ly.root] = make(map[common.Hash]struct{})
+		c.diffLayerParent[ly.root][ly.root] = struct{}{}
+	}
+	diffMultiVersionCacheLengthGauge.Update(c.cacheItemNumber)
+}
+
+func (c *MultiVersionSnapshotCache) loopDelayGC() {
+	if c == nil {
+		return
+	}
+
+	gcTicker := time.NewTicker(time.Second * 1)
+	defer gcTicker.Stop()
+	for {
+		select {
+		case <-gcTicker.C:
+			c.lock.RLock()
+			deltaQueueLen := len(c.deltaRemoveQueue)
+			c.lock.RUnlock()
+			if deltaQueueLen > 500 {
+				c.lock.Lock()
+				gcDifflayer := c.deltaRemoveQueue[batchGCNumber]
+				if gcDifflayer.diffLayerID > c.minVersion {
+					c.minVersion = gcDifflayer.diffLayerID
+				}
+				log.Info("Delay remove difflayer from snapshot multiversion cache",
+					"root", gcDifflayer.root,
+					"version_id", gcDifflayer.diffLayerID,
+					"current_cache_item_number", c.cacheItemNumber,
+					"deleted_difflayer_number", deltaQueueLen,
+					"min_version", c.minVersion)
+
+				for aHash, multiVersionDestructList := range c.destructCache {
+					for i := 0; i < len(multiVersionDestructList); i++ {
+						if multiVersionDestructList[i].version < c.minVersion {
+							//log.Info("Remove destruct from cache",
+							//	"cache_account_hash", aHash,
+							//	"cache_version", multiVersionDestructList[i].version,
+							//	"cache_root", multiVersionDestructList[i].root,
+							//	"min_version", c.minVersion,
+							//	"gc_diff_root", gcDifflayer.root,
+							//	"gc_diff_version", gcDifflayer.diffLayerID)
+							multiVersionDestructList = append(multiVersionDestructList[:i], multiVersionDestructList[i+1:]...)
+							i--
+							c.cacheItemNumber--
+						}
+					}
+					if len(multiVersionDestructList) == 0 {
+						delete(c.destructCache, aHash)
+					} else {
+						c.destructCache[aHash] = multiVersionDestructList
+					}
+				}
+
+				for aHash, multiVersionAccoutList := range c.accountDataCache {
+					for i := 0; i < len(multiVersionAccoutList); i++ {
+						if multiVersionAccoutList[i].version < c.minVersion {
+							//log.Info("Remove account from cache",
+							//	"cache_account_hash", aHash,
+							//	"cache_version", multiVersionAccoutList[i].version,
+							//	"cache_root", multiVersionAccoutList[i].root,
+							//	"cache_data_len", len(multiVersionAccoutList[i].data),
+							//	"min_version", c.minVersion,
+							//	"gc_diff_root", gcDifflayer.root,
+							//	"gc_diff_version", gcDifflayer.diffLayerID)
+							multiVersionAccoutList = append(multiVersionAccoutList[:i], multiVersionAccoutList[i+1:]...)
+							i--
+							c.cacheItemNumber--
+						}
+					}
+					if len(multiVersionAccoutList) == 0 {
+						delete(c.accountDataCache, aHash)
+					} else {
+						c.accountDataCache[aHash] = multiVersionAccoutList
+					}
+				}
+				for aHash := range c.storageDataCache {
+					for sHash, multiVersionStorageList := range c.storageDataCache[aHash] {
+						for i := 0; i < len(multiVersionStorageList); i++ {
+							if multiVersionStorageList[i].version < c.minVersion {
+								//log.Info("Remove storage from cache",
+								//	"cache_account_hash", aHash,
+								//	"cache_storage_hash", sHash,
+								//	"cache_version", multiVersionStorageList[i].version,
+								//	"cache_root", multiVersionStorageList[i].root,
+								//	"cache_data_len", len(multiVersionStorageList[i].data),
+								//	"min_version", c.minVersion,
+								//	"gc_diff_root", gcDifflayer.root,
+								//	"gc_diff_version", gcDifflayer.diffLayerID)
+								multiVersionStorageList = append(multiVersionStorageList[:i], multiVersionStorageList[i+1:]...)
+								i--
+								c.cacheItemNumber--
+							}
+						}
+						if len(multiVersionStorageList) == 0 {
+							delete(c.storageDataCache[aHash], sHash)
+						} else {
+							c.storageDataCache[aHash][sHash] = multiVersionStorageList
+						}
+					}
+					if len(c.storageDataCache[aHash]) == 0 {
+						delete(c.storageDataCache, aHash)
+					}
+				}
+
+				for i := 0; i <= batchGCNumber; i++ {
+					toGCDifflayerRoot := c.deltaRemoveQueue[i].root
+					delete(c.diffLayerParent, toGCDifflayerRoot)
+					for _, v := range c.diffLayerParent {
+						delete(v, toGCDifflayerRoot)
+					}
+				}
+
+				diffMultiVersionCacheLengthGauge.Update(c.cacheItemNumber)
+				c.deltaRemoveQueue = c.deltaRemoveQueue[batchGCNumber+1:]
+				c.lock.Unlock()
+			} else {
+				log.Info("Skip delay gc due to less difflayer in queue", "deleted_difflayer_number", deltaQueueLen)
+			}
+		}
+	}
+}
+
+func (c *MultiVersionSnapshotCache) Remove(ly *diffLayer) {
+	if c == nil || ly == nil {
+		return
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.deltaRemoveQueue = append(c.deltaRemoveQueue, ly)
+}
+
+// QueryAccount return tuple(data-slice, need-try-disklayer, error)
+func (c *MultiVersionSnapshotCache) QueryAccount(version uint64, rootHash common.Hash, ahash common.Hash) ([]byte, bool, error) {
+	if c == nil {
+		return nil, false, fmt.Errorf("not found, need try difflayer")
+	}
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	var (
+		queryAccountItem  *accountCacheItem
+		queryDestructItem *destructCacheItem
+	)
+
+	{
+		if multiVersionItems, exist := c.accountDataCache[ahash]; exist && len(multiVersionItems) != 0 {
+			//log.Info("Try query account cache",
+			//	"query_version", version,
+			//	"query_root_hash", rootHash,
+			//	"query_account_hash", ahash,
+			//	"multi_version_cache_len", len(multiVersionItems))
+			for i := len(multiVersionItems) - 1; i >= 0; i-- {
+				if multiVersionItems[i].version <= version &&
+					multiVersionItems[i].version > c.minVersion &&
+					c.checkParent(rootHash, multiVersionItems[i].root) {
+					queryAccountItem = multiVersionItems[i]
+					//log.Info("Account hit account cache",
+					//	"query_version", version,
+					//	"query_root_hash", rootHash,
+					//	"query_account_hash", ahash,
+					//	"hit_version", queryAccountItem.version,
+					//	"hit_root_hash", queryAccountItem.root)
+					break
+				}
+				//log.Info("Try hit account cache",
+				//	"query_version", version,
+				//	"query_root_hash", rootHash,
+				//	"query_account_hash", ahash,
+				//	"try_hit_version", multiVersionItems[i].version,
+				//	"try_hit_root_hash", multiVersionItems[i].root,
+				//	"check_version", multiVersionItems[i].version > c.minVersion,
+				//	"check_parent", c.checkParent(rootHash, multiVersionItems[i].root),
+				//	"check_data_len", len(multiVersionItems[i].data))
+			}
+		}
+	}
+
+	{
+		if multiVersionItems, exist := c.destructCache[ahash]; exist && len(multiVersionItems) != 0 {
+			//log.Info("Try query destruct cache",
+			//	"query_version", version,
+			//	"query_root_hash", rootHash,
+			//	"query_account_hash", ahash,
+			//	"multi_version_cache_len", len(multiVersionItems))
+			for i := len(multiVersionItems) - 1; i >= 0; i-- {
+				if multiVersionItems[i].version <= version &&
+					multiVersionItems[i].version > c.minVersion &&
+					c.checkParent(rootHash, multiVersionItems[i].root) {
+					queryDestructItem = multiVersionItems[i]
+					//log.Info("Account hit destruct cache",
+					//	"query_version", version,
+					//	"query_root_hash", rootHash,
+					//	"query_account_hash", ahash,
+					//	"hit_version", queryDestructItem.version,
+					//	"hit_root_hash", queryDestructItem.root)
+					break
+				}
+				//log.Info("Try hit destruct cache",
+				//	"query_version", version,
+				//	"query_root_hash", rootHash,
+				//	"query_account_hash", ahash,
+				//	"hit_version", multiVersionItems[i].version,
+				//	"hit_root_hash", multiVersionItems[i].root)
+			}
+		}
+	}
+	if queryAccountItem != nil && queryDestructItem == nil {
+		return queryAccountItem.data, false, nil // founded
+	}
+
+	if queryAccountItem == nil && queryDestructItem != nil {
+		return nil, false, nil // deleted
+	}
+
+	if queryAccountItem == nil && queryDestructItem == nil {
+		return nil, true, nil
+	}
+
+	// queryAccountItem != nil && queryDestructItem != nil
+	if queryAccountItem.version >= queryDestructItem.version {
+		return queryAccountItem.data, false, nil // founded
+	} else {
+		return nil, false, nil // deleted
+	}
+}
+
+// QueryStorage return tuple(data-slice, need-try-disklayer, error)
+func (c *MultiVersionSnapshotCache) QueryStorage(version uint64, rootHash common.Hash, ahash common.Hash, shash common.Hash) ([]byte, bool, error) {
+	if c == nil {
+		return nil, false, fmt.Errorf("not found, need try difflayer")
+	}
+
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	var (
+		queryStorageItem  *storageCacheItem
+		queryDestructItem *destructCacheItem
+	)
+
+	{
+		if _, exist := c.storageDataCache[ahash]; exist {
+			if multiVersionItems, exist2 := c.storageDataCache[ahash][shash]; exist2 && len(multiVersionItems) != 0 {
+				//log.Info("Try query storage cache",
+				//	"query_version", version,
+				//	"query_root_hash", rootHash,
+				//	"query_account_hash", ahash,
+				//	"query_storage_hash", shash,
+				//	"multi_version_cache_len", len(multiVersionItems))
+				for i := len(multiVersionItems) - 1; i >= 0; i-- {
+					if multiVersionItems[i].version <= version &&
+						multiVersionItems[i].version > c.minVersion &&
+						c.checkParent(rootHash, multiVersionItems[i].root) {
+						queryStorageItem = multiVersionItems[i]
+						//log.Info("Account hit storage cache",
+						//	"query_version", version,
+						//	"query_root_hash", rootHash,
+						//	"query_account_hash", ahash,
+						//	"query_storage_hash", shash,
+						//	"hit_version", queryStorageItem.version,
+						//	"hit_root_hash", queryStorageItem.root)
+						break
+					}
+					//log.Info("Try hit storage cache",
+					//	"query_version", version,
+					//	"query_root_hash", rootHash,
+					//	"query_account_hash", ahash,
+					//	"query_storage_hash", shash,
+					//	"hit_version", multiVersionItems[i].version,
+					//	"hit_root_hash", multiVersionItems[i].root)
+				}
+			}
+		}
+	}
+
+	{
+		if multiVersionItems, exist := c.destructCache[ahash]; exist && len(multiVersionItems) != 0 {
+			//log.Info("Try query destruct cache",
+			//	"query_version", version,
+			//	"query_root_hash", rootHash,
+			//	"query_account_hash", ahash,
+			//	"query_storage_hash", shash,
+			//	"multi_version_cache_len", len(multiVersionItems))
+			for i := len(multiVersionItems) - 1; i >= 0; i-- {
+				if multiVersionItems[i].version <= version &&
+					multiVersionItems[i].version > c.minVersion &&
+					c.checkParent(rootHash, multiVersionItems[i].root) {
+					queryDestructItem = multiVersionItems[i]
+					//log.Info("Account hit destruct cache",
+					//	"query_version", version,
+					//	"query_root_hash", rootHash,
+					//	"query_account_hash", ahash,
+					//	"query_storage_hash", shash,
+					//	"hit_version", queryDestructItem.version,
+					//	"hit_root_hash", queryDestructItem.root)
+					break
+				}
+				//log.Info("Try hit destruct cache",
+				//	"query_version", version,
+				//	"query_root_hash", rootHash,
+				//	"query_account_hash", ahash,
+				//	"query_storage_hash", shash,
+				//	"hit_version", multiVersionItems[i].version,
+				//	"hit_root_hash", multiVersionItems[i].root)
+			}
+		}
+	}
+
+	if queryStorageItem != nil && queryDestructItem == nil {
+		return queryStorageItem.data, false, nil // founded
+	}
+
+	if queryStorageItem == nil && queryDestructItem != nil {
+		return nil, false, nil // deleted
+	}
+
+	if queryStorageItem == nil && queryDestructItem == nil {
+		return nil, true, nil // not founded and need try disklayer
+	}
+
+	// queryStorageItem != nil && queryDestructItem != nil
+	if queryStorageItem.version >= queryDestructItem.version {
+		return queryStorageItem.data, false, nil // founded
+	} else {
+		return nil, false, nil // deleted
+	}
+}

core/state/snapshot/snapshot.go

@@ -365,6 +365,9 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m
 	}
 	snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage)
 
+	log.Info("Add cache due to new difflayer", "diff_root", snap.root, "diff_version", snap.diffLayerID)
+	snap.AddToCache()
+
 	// Save the new snapshot for later
 	t.lock.Lock()
 	defer t.lock.Unlock()
@@ -412,6 +415,13 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
 		base := diffToDisk(diff.flatten().(*diffLayer))
 		diff.lock.RUnlock()
 
+		for _, ly := range t.layers {
+			if diff, ok := ly.(*diffLayer); ok {
+				log.Info("Cleanup cache due to layers=0", "diff_root", diff.root, "diff_version", diff.diffLayerID)
+				diff.RemoveFromCache()
+			}
+		}
+
 		// Replace the entire snapshot tree with the flat base
 		t.layers = map[common.Hash]snapshot{base.root: base}
 		return nil
@@ -428,6 +438,12 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
 	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
+		if df, exist := t.layers[root]; exist {
+			if diff, ok := df.(*diffLayer); ok {
+				log.Info("Cleanup cache due to reorg or stale", "diff_root", diff.root, "diff_version", diff.diffLayerID)
+				diff.RemoveFromCache()
+			}
+		}
 		delete(t.layers, root)
 		for _, child := range children[root] {
 			remove(child)
@@ -495,6 +511,10 @@ func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer {
 		flattened := parent.flatten().(*diffLayer)
 		t.layers[flattened.root] = flattened
 
+		// the new flatten difflayer will cause multiple versions of the cache to be out of order. so need resorted multi-version cache.
+		log.Info("Add cache due to flatten", "diff_root", flattened.root, "diff_version", flattened.diffLayerID)
+		flattened.AddToCache()
+
 		// Invoke the hook if it's registered. Ugly hack.
 		if t.onFlatten != nil {
 			t.onFlatten()
@@ -661,6 +681,9 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 		res.genAbort = make(chan chan *generatorStats)
 		go res.generate(stats)
 	}
+
+	log.Info("Cleanup cache due to bottom difflayer is eaten by disklayer", "diff_root", bottom.root, "diff_version", bottom.diffLayerID)
+	bottom.RemoveFromCache()
 	return res
 }