diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index aa05c5dfe7..cdc98221ba 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -50,6 +50,9 @@ func TestNodeIteratorCoverage(t *testing.T) { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } + if bytes.HasPrefix(key, []byte("accounts-bloom")) { + continue + } if _, ok := hashes[common.BytesToHash(key)]; !ok { t.Errorf("state entry not reported %x", key) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 2dd833f584..bcdc48d048 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,8 +18,10 @@ package state import ( + "bytes" "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/vm" @@ -28,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + boom "github.com/tylertreat/BoomFilters" ) // The starting nonce determines the default nonce when new accounts are being @@ -46,6 +49,11 @@ type StateDB struct { // This map caches canon state accounts. all map[common.Address]Account + // This filter is used to quickly check if an account exists, + // avoiding expensive database lookups. + accountBloom *boom.ScalableBloomFilter + accountBloomDirty bool + // This map holds 'live' objects, which will get modified // while processing a state transition. stateObjects map[common.Address]*StateObject @@ -94,6 +102,7 @@ func (self *StateDB) Reset(root common.Hash) error { db: self.db, trie: tr, all: all, + accountBloom: self.accountBloom, stateObjects: make(map[common.Address]*StateObject), refund: new(big.Int), logs: make(map[common.Hash]vm.Logs), @@ -270,14 +279,16 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje } return obj } - // Use cached account data from the canon state if possible. if data, ok := self.all[addr]; ok { obj := NewObject(addr, data) self.SetStateObject(obj) return obj } - + // Avoid surely non-existing accounts + if !self.getAccountBloom().Test(self.trie.HashKey(addr[:])) { + return nil + } // Load the object from the database. enc := self.trie.Get(addr[:]) if len(enc) == 0 { @@ -321,6 +332,9 @@ func (self *StateDB) newStateObject(addr common.Address) *StateObject { obj.dirty = true obj.SetNonce(StartingNonce) self.stateObjects[addr] = obj + if !self.getAccountBloom().TestAndAdd(self.trie.HashKey(addr[:])) { + self.accountBloomDirty = true + } return obj } @@ -335,7 +349,6 @@ func (self *StateDB) CreateStateObject(addr common.Address) *StateObject { if so != nil { newSo.data.Balance = so.data.Balance } - return newSo } @@ -352,6 +365,8 @@ func (self *StateDB) Copy() *StateDB { state, _ := New(common.Hash{}, self.db) state.trie = self.trie state.all = self.all + state.accountBloom = self.accountBloom + state.accountBloomDirty = self.accountBloomDirty for addr, stateObject := range self.stateObjects { if stateObject.dirty { state.stateObjects[addr] = stateObject.Copy(self.db) @@ -373,6 +388,8 @@ func (self *StateDB) Set(state *StateDB) { self.trie = state.trie self.stateObjects = state.stateObjects self.all = state.all + self.accountBloom = state.accountBloom + self.accountBloomDirty = state.accountBloomDirty self.refund = state.refund self.logs = state.logs @@ -471,10 +488,67 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) } stateObject.dirty = false } - // Write trie changes. + // Write account bloom and trie changes + if s.accountBloomDirty { + if err := s.writeAccountsBloom(dbw); err != nil { + return common.Hash{}, err + } + s.accountBloomDirty = false + } return s.trie.CommitTo(dbw) } func (self *StateDB) Refunds() *big.Int { return self.refund } + +var accountBloomKey = []byte("accounts-bloom") + +// getAccountBloom returns the account existential bloom filter, lazy loading it +// from disk if needed. +func (self *StateDB) getAccountBloom() *boom.ScalableBloomFilter { + // Short circuit lookup if already loaded from the database + if self.accountBloom != nil { + return self.accountBloom + } + // If the filter's not yet cached, load it from the database + bloom, _ := self.db.Get(accountBloomKey) + if len(bloom) > 0 { + filter := new(boom.ScalableBloomFilter) + if _, err := filter.ReadFrom(bytes.NewReader(bloom)); err != nil { + return nil + } + self.accountBloom = filter + return self.accountBloom + } + // If the database does not yet contain a bloom filter, generate a new one + glog.V(logger.Info).Infof("generating accounts bloom, please wait...") + + filter := boom.NewDefaultScalableBloomFilter(0.01) + start, pref := time.Now(), byte(0x00) + + it := self.trie.Iterator() + for it.Next() { + if pref != it.Key[0] { + pref = it.Key[0] + glog.V(logger.Info).Infof("generating accounts bloom, at %.2f%%, please wait...", float64(pref)/2.56) + } + filter.Add(it.Key) + } + glog.V(logger.Info).Infof("accounts bloom generated in %v", time.Since(start)) + + self.accountBloom, self.accountBloomDirty = filter, true + return self.accountBloom +} + +// writeAccountsBloom serializes the account bloom filter used for existence checks. +func (self *StateDB) writeAccountsBloom(db trie.DatabaseWriter) error { + if self.accountBloom == nil { + return nil + } + buffer := new(bytes.Buffer) + if _, err := self.accountBloom.WriteTo(buffer); err != nil { + return err + } + return db.Put(accountBloomKey, buffer.Bytes()) +} diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 1d027c1027..73c1735214 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -46,7 +46,7 @@ type SecureTrie struct { secKeyCache map[string][]byte } -// NewSecure creates a trie with an existing root node from db. +// NewSecure creFGetates a trie with an existing root node from db. // // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil @@ -138,6 +138,11 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { return key } +// HashKey returns the sha3 image of a key. +func (t *SecureTrie) HashKey(key []byte) []byte { + return common.CopyBytes(t.hashKey(key)) +} + // Commit writes all nodes and the secure hash pre-images to the trie's database. // Nodes are stored with their sha3 hash as the key. //