mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
core/state/snapshot: use local bloom filters
This commit is contained in:
Martin Holst Swende
parent
4373220585
commit
244ff6e96b
6 changed files with 90 additions and 52 deletions
|
|
@ -27,7 +27,7 @@ import (
|
|||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/steakknife/bloomfilter"
|
||||
"github.com/holiman/bloomfilter"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -110,7 +110,8 @@ type diffLayer struct {
|
|||
storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
|
||||
storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrival. one per account (nil means deleted)
|
||||
|
||||
diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
|
||||
diffed *bloomfilter.Filter // Bloom filter tracking local diffed items
|
||||
cumulative *bloomfilter.Filter // cumulumative bloom filter -- by default set to nil
|
||||
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
|
@ -169,34 +170,24 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s
|
|||
accountData: accounts,
|
||||
storageData: storage,
|
||||
}
|
||||
switch parent := parent.(type) {
|
||||
case *diskLayer:
|
||||
dl.rebloom(parent, true)
|
||||
case *diffLayer:
|
||||
dl.rebloom(parent.origin, true)
|
||||
default:
|
||||
panic("unknown parent type")
|
||||
}
|
||||
dl.initBloom()
|
||||
return dl
|
||||
}
|
||||
|
||||
// rebloom discards the layer's current bloom and rebuilds it from scratch based
|
||||
// on the parent's and the local diffs.
|
||||
func (dl *diffLayer) rebloom(origin *diskLayer, creation bool) {
|
||||
// initBloom builds the layer's bloom and calculates memory consumption
|
||||
func (dl *diffLayer) initBloom() {
|
||||
dl.lock.Lock()
|
||||
defer dl.lock.Unlock()
|
||||
|
||||
defer func(start time.Time) {
|
||||
snapshotBloomIndexTimer.Update(time.Since(start))
|
||||
}(time.Now())
|
||||
|
||||
// Inject the new origin that triggered the rebloom
|
||||
dl.origin = origin
|
||||
|
||||
// Retrieve the parent bloom or create a fresh empty one
|
||||
// We need to use the same keys as the parent bloom if we want to be able
|
||||
// to make union later
|
||||
if parent, ok := dl.parent.(*diffLayer); ok {
|
||||
parent.lock.RLock()
|
||||
dl.diffed, _ = parent.diffed.Copy()
|
||||
dl.diffed, _ = parent.diffed.NewCompatible()
|
||||
parent.lock.RUnlock()
|
||||
} else {
|
||||
dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
|
||||
|
|
@ -218,9 +209,7 @@ func (dl *diffLayer) rebloom(origin *diskLayer, creation bool) {
|
|||
nHashes++
|
||||
}
|
||||
dl.memory = dataSize + nHashes*uint64(common.HashLength)
|
||||
if creation {
|
||||
snapshotDirtyAccountWriteMeter.Mark(int64(dataSize))
|
||||
}
|
||||
snapshotDirtyAccountWriteMeter.Mark(int64(dataSize))
|
||||
|
||||
dataSize, nHashes = uint64(0), uint64(0)
|
||||
for accountHash, slots := range dl.storageData {
|
||||
|
|
@ -235,16 +224,39 @@ func (dl *diffLayer) rebloom(origin *diskLayer, creation bool) {
|
|||
}
|
||||
}
|
||||
dl.memory += dataSize + nHashes*uint64(common.HashLength)
|
||||
if creation {
|
||||
snapshotDirtyStorageWriteMeter.Mark(int64(dataSize))
|
||||
snapshotDirtyStorageWriteMeter.Mark(int64(dataSize))
|
||||
}
|
||||
|
||||
// Prepare prepares the difflayer for execution, and creates the cumulative
|
||||
// bloom
|
||||
func (dl *diffLayer) Prepare(origin *diskLayer) {
|
||||
dl.lock.Lock()
|
||||
dl.cumulative, _ = dl.diffed.Copy()
|
||||
layer := dl
|
||||
for {
|
||||
if parent, ok := layer.parent.(*diffLayer); ok {
|
||||
parent.lock.RLock()
|
||||
dl.cumulative.UnionInPlace(parent.diffed)
|
||||
parent.lock.RUnlock()
|
||||
layer = parent
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
dl.origin = origin
|
||||
dl.lock.Unlock()
|
||||
// Calculate the current false positive rate and update the error rate meter.
|
||||
// This is a bit cheating because subsequent layers will overwrite it, but it
|
||||
// should be fine, we're only interested in ballpark figures.
|
||||
k := float64(dl.diffed.K())
|
||||
n := float64(dl.diffed.N())
|
||||
m := float64(dl.diffed.M())
|
||||
k := float64(dl.cumulative.K())
|
||||
n := float64(dl.cumulative.N())
|
||||
m := float64(dl.cumulative.M())
|
||||
snapshotBloomErrorGauge.Update(math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k))
|
||||
|
||||
}
|
||||
|
||||
func (dl *diffLayer) Release() {
|
||||
dl.lock.Lock()
|
||||
dl.cumulative = nil
|
||||
dl.lock.Unlock()
|
||||
}
|
||||
|
||||
// Root returns the root hash for which this snapshot was made.
|
||||
|
|
@ -286,9 +298,9 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
|
|||
// Check the bloom filter first whether there's even a point in reaching into
|
||||
// all the maps in all the layers below
|
||||
dl.lock.RLock()
|
||||
hit := dl.diffed.Contains(accountBloomHasher(hash))
|
||||
hit := dl.cumulative.Contains(accountBloomHasher(hash))
|
||||
if !hit {
|
||||
hit = dl.diffed.Contains(destructBloomHasher(hash))
|
||||
hit = dl.cumulative.Contains(destructBloomHasher(hash))
|
||||
}
|
||||
dl.lock.RUnlock()
|
||||
|
||||
|
|
@ -346,9 +358,9 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
|
|||
// Check the bloom filter first whether there's even a point in reaching into
|
||||
// all the maps in all the layers below
|
||||
dl.lock.RLock()
|
||||
hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash})
|
||||
hit := dl.cumulative.Contains(storageBloomHasher{accountHash, storageHash})
|
||||
if !hit {
|
||||
hit = dl.diffed.Contains(destructBloomHasher(accountHash))
|
||||
hit = dl.cumulative.Contains(destructBloomHasher(accountHash))
|
||||
}
|
||||
dl.lock.RUnlock()
|
||||
|
||||
|
|
@ -457,6 +469,7 @@ func (dl *diffLayer) flatten() snapshot {
|
|||
parent.storageData[accountHash] = comboData
|
||||
}
|
||||
// Return the combo parent
|
||||
parent.diffed.UnionInPlace(dl.diffed)
|
||||
return &diffLayer{
|
||||
parent: parent.parent,
|
||||
origin: parent.origin,
|
||||
|
|
@ -465,7 +478,7 @@ func (dl *diffLayer) flatten() snapshot {
|
|||
accountData: parent.accountData,
|
||||
storageData: parent.storageData,
|
||||
storageList: make(map[common.Hash][]common.Hash),
|
||||
diffed: dl.diffed,
|
||||
diffed: parent.diffed,
|
||||
memory: parent.memory + dl.memory,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -164,3 +164,6 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
|
|||
func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
|
||||
return newDiffLayer(dl, blockHash, destructs, accounts, storage)
|
||||
}
|
||||
|
||||
func (dl *diskLayer) Prepare(*diskLayer) {}
|
||||
func (dl *diskLayer) Release() {}
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ type Snapshot interface {
|
|||
// Storage directly retrieves the storage data associated with a particular hash,
|
||||
// within a particular account.
|
||||
Storage(accountHash, storageHash common.Hash) ([]byte, error)
|
||||
Release()
|
||||
}
|
||||
|
||||
// snapshot is the internal version of the snapshot data layer that supports some
|
||||
|
|
@ -138,6 +139,8 @@ type snapshot interface {
|
|||
|
||||
// AccountIterator creates an account iterator over an arbitrary layer.
|
||||
AccountIterator(seek common.Hash) AccountIterator
|
||||
|
||||
Prepare(*diskLayer)
|
||||
}
|
||||
|
||||
// SnapshotTree is an Ethereum state snapshot tree. It consists of one persistent
|
||||
|
|
@ -150,11 +153,12 @@ type snapshot interface {
|
|||
// storage data to avoid expensive multi-level trie lookups; and to allow sorted,
|
||||
// cheap iteration of the account/storage tries for sync aid.
|
||||
type Tree struct {
|
||||
diskdb ethdb.KeyValueStore // Persistent database to store the snapshot
|
||||
triedb *trie.Database // In-memory cache to access the trie through
|
||||
cache int // Megabytes permitted to use for read caches
|
||||
layers map[common.Hash]snapshot // Collection of all known layers
|
||||
lock sync.RWMutex
|
||||
diskdb ethdb.KeyValueStore // Persistent database to store the snapshot
|
||||
triedb *trie.Database // In-memory cache to access the trie through
|
||||
cache int // Megabytes permitted to use for read caches
|
||||
layers map[common.Hash]snapshot // Collection of all known layers
|
||||
lock sync.RWMutex
|
||||
diskLayer *diskLayer // The underlying disklayer
|
||||
}
|
||||
|
||||
// New attempts to load an already existing snapshot from a persistent key-value
|
||||
|
|
@ -186,6 +190,9 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm
|
|||
for head != nil {
|
||||
snap.layers[head.Root()] = head
|
||||
head = head.Parent()
|
||||
// TODO(@holiman), check if we need to add this back:
|
||||
//case *diskLayer:
|
||||
// snap.diskLayer = self
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
|
@ -211,6 +218,16 @@ func (t *Tree) waitBuild() {
|
|||
}
|
||||
}
|
||||
|
||||
func (t *Tree) PrepareSnapshot(blockRoot common.Hash) Snapshot {
|
||||
t.lock.RLock()
|
||||
defer t.lock.RUnlock()
|
||||
if snap := t.layers[blockRoot]; snap != nil {
|
||||
snap.Prepare(t.diskLayer)
|
||||
return snap
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Snapshot retrieves a snapshot belonging to the given block root, or nil if no
|
||||
// snapshot is maintained for that block.
|
||||
func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot {
|
||||
|
|
@ -278,6 +295,7 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
|
|||
|
||||
// Replace the entire snapshot tree with the flat base
|
||||
t.layers = map[common.Hash]snapshot{base.root: base}
|
||||
t.diskLayer = base
|
||||
return nil
|
||||
|
||||
case 1:
|
||||
|
|
@ -291,6 +309,7 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
|
|||
bottom = diff.flatten().(*diffLayer)
|
||||
if bottom.memory >= aggregatorMemoryLimit {
|
||||
base = diffToDisk(bottom)
|
||||
t.diskLayer = base
|
||||
}
|
||||
diff.lock.RUnlock()
|
||||
|
||||
|
|
@ -305,6 +324,9 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
|
|||
default:
|
||||
// Many layers requested to be retained, cap normally
|
||||
persisted = t.cap(diff, layers)
|
||||
if persisted != nil {
|
||||
t.diskLayer = persisted
|
||||
}
|
||||
}
|
||||
// Remove any layer that is stale or links into a stale layer
|
||||
children := make(map[common.Hash][]common.Hash)
|
||||
|
|
@ -328,18 +350,6 @@ func (t *Tree) Cap(root common.Hash, layers int) error {
|
|||
}
|
||||
}
|
||||
// If the disk layer was modified, regenerate all the cummulative blooms
|
||||
if persisted != nil {
|
||||
var rebloom func(root common.Hash)
|
||||
rebloom = func(root common.Hash) {
|
||||
if diff, ok := t.layers[root].(*diffLayer); ok {
|
||||
diff.rebloom(persisted, false)
|
||||
}
|
||||
for _, child := range children[root] {
|
||||
rebloom(child)
|
||||
}
|
||||
}
|
||||
rebloom(persisted.root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -591,8 +601,10 @@ func (t *Tree) Rebuild(root common.Hash) {
|
|||
// Start generating a new snapshot from scratch on a backgroung thread. The
|
||||
// generator will run a wiper first if there's not one running right now.
|
||||
log.Info("Rebuilding state snapshot")
|
||||
diskLayer := generateSnapshot(t.diskdb, t.triedb, t.cache, root, wiper)
|
||||
t.diskLayer = diskLayer
|
||||
t.layers = map[common.Hash]snapshot{
|
||||
root: generateSnapshot(t.diskdb, t.triedb, t.cache, root, wiper),
|
||||
root: diskLayer,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error)
|
|||
journal: newJournal(),
|
||||
}
|
||||
if sdb.snaps != nil {
|
||||
if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil {
|
||||
if sdb.snap = sdb.snaps.PrepareSnapshot(root); sdb.snap != nil {
|
||||
sdb.snapDestructs = make(map[common.Hash]struct{})
|
||||
sdb.snapAccounts = make(map[common.Hash][]byte)
|
||||
sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte)
|
||||
|
|
@ -862,7 +862,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
|
|||
if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil {
|
||||
log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err)
|
||||
}
|
||||
if err := s.snaps.Cap(root, 127); err != nil { // Persistent layer is 128th, the last available trie
|
||||
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 127, "err", err)
|
||||
}
|
||||
}
|
||||
s.snap.Release()
|
||||
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
|
||||
}
|
||||
return root, err
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -33,6 +33,7 @@ require (
|
|||
github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277
|
||||
github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad
|
||||
github.com/holiman/bloomfilter v0.0.0-20191204204232-5fc8e9365fc2
|
||||
github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3
|
||||
github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883
|
||||
github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
|
||||
|
|
@ -57,6 +58,7 @@ require (
|
|||
github.com/stretchr/testify v1.4.0
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
|
||||
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
|
||||
github.com/umbracle/fastrlp v0.0.0-20191017143648-86584926e68c
|
||||
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
|
||||
golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -99,6 +99,8 @@ github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKx
|
|||
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad h1:eMxs9EL0PvIGS9TTtxg4R+JxuPGav82J8rA+GFnY7po=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/holiman/bloomfilter v0.0.0-20191204204232-5fc8e9365fc2 h1:W++/MG1xoR0V8YM2H47YC/G/DnZKODb0TmxY/GBcEac=
|
||||
github.com/holiman/bloomfilter v0.0.0-20191204204232-5fc8e9365fc2/go.mod h1:+E92jn6hSglI3YfM9VLe+w2tHKtSL9l83iBiZS7/4Lw=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3 h1:DqD8eigqlUm0+znmx7zhL0xvTW3+e1jCekJMfBUADWI=
|
||||
|
|
@ -189,6 +191,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+
|
|||
github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4=
|
||||
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
|
||||
github.com/umbracle/fastrlp v0.0.0-20191017143648-86584926e68c h1:KCbyOp0afQzDTGARP+qj9HpZVk79WQaQtAO+4kMy/E4=
|
||||
github.com/umbracle/fastrlp v0.0.0-20191017143648-86584926e68c/go.mod h1:TBFAIaLMusGh1X7Pd3trJTjLsOmNErpPV9/J7BY9lSA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
|
||||
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
|
||||
|
|
|
|||
Loading…
Reference in a new issue