From e00520297735677ffd1d8502131bdade18e7bbd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sun, 25 Sep 2016 10:01:13 +0300 Subject: [PATCH] core, tests: track dirty objects in separate map --- core/blockchain.go | 12 +++- core/state/dump.go | 2 +- core/state/iterator_test.go | 2 +- core/state/state_object.go | 43 +++++++----- core/state/state_test.go | 11 ++- core/state/statedb.go | 130 ++++++++++++++++++++---------------- core/state/statedb_test.go | 4 ++ tests/state_test_util.go | 4 +- tests/util.go | 4 +- tests/vm_test_util.go | 4 +- 10 files changed, 124 insertions(+), 92 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index be336f6544..a5f146a2d3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -197,7 +197,15 @@ func (self *BlockChain) loadLastState() error { self.currentFastBlock = block } } - // Issue a status log and return + // Initialize a statedb cache to ensure singleton account bloom filter generation + statedb, err := state.New(self.currentBlock.Root(), self.chainDb) + if err != nil { + return err + } + self.stateCache = statedb + self.stateCache.GetAccount(common.Address{}) + + // Issue a status log for the user headerTd := self.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) blockTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) fastTd := self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()) @@ -894,8 +902,6 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Create a new statedb using the parent block and report an // error if it fails. switch { - case self.stateCache == nil: - self.stateCache, err = state.New(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), self.chainDb) case i == 0: err = self.stateCache.Reset(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) default: diff --git a/core/state/dump.go b/core/state/dump.go index 849f3548f0..5140449c70 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -52,7 +52,7 @@ func (self *StateDB) RawDump() Dump { panic(err) } - obj := NewObject(common.BytesToAddress(addr), data) + obj := NewObject(common.BytesToAddress(addr), data, nil) account := DumpAccount{ Balance: data.Balance.String(), Nonce: data.Nonce, diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index cdc98221ba..4f323a3503 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -50,7 +50,7 @@ func TestNodeIteratorCoverage(t *testing.T) { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } - if bytes.HasPrefix(key, []byte("accounts-bloom")) { + if bytes.Equal(key, []byte("accounts-bloom")) { continue } if _, ok := hashes[common.BytesToHash(key)]; !ok { diff --git a/core/state/state_object.go b/core/state/state_object.go index 81ff205b86..7c4bbced44 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -82,10 +82,10 @@ type StateObject struct { // Cache flags. // When an object is marked for deletion it will be delete from the trie // during the "update" phase of the state transition - dirty bool // true if anything has changed dirtyCode bool // true if the code was updated remove bool deleted bool + onDirty func(addr common.Address) // Callback method to mark a state object newly dirty } // Account is the Ethereum consensus representation of accounts. @@ -100,14 +100,14 @@ type Account struct { } // NewObject creates a state object. -func NewObject(address common.Address, data Account) *StateObject { +func NewObject(address common.Address, data Account, onDirty func(addr common.Address)) *StateObject { if data.Balance == nil { data.Balance = new(big.Int) } if data.CodeHash == nil { data.CodeHash = emptyCodeHash } - return &StateObject{address: address, data: data, storage: make(Storage)} + return &StateObject{address: address, data: data, storage: make(Storage), onDirty: onDirty} } // EncodeRLP implements rlp.Encoder. @@ -124,8 +124,10 @@ func (self *StateObject) setError(err error) { func (self *StateObject) MarkForDeletion() { self.remove = true - self.dirty = true - + if self.onDirty != nil { + self.onDirty(self.Address()) + self.onDirty = nil + } if glog.V(logger.Core) { glog.Infof("%x: #%d %v X\n", self.Address(), self.Nonce(), self.Balance()) } @@ -162,7 +164,10 @@ func (self *StateObject) GetState(db trie.Database, key common.Hash) common.Hash // SetState updates a value in account storage. func (self *StateObject) SetState(key, value common.Hash) { self.storage[key] = value - self.dirty = true + if self.onDirty != nil { + self.onDirty(self.Address()) + self.onDirty = nil + } } // updateTrie writes cached storage modifications into the object's storage trie. @@ -222,22 +227,23 @@ func (c *StateObject) SubBalance(amount *big.Int) { } } -func (c *StateObject) SetBalance(amount *big.Int) { - c.data.Balance = amount - c.dirty = true +func (self *StateObject) SetBalance(amount *big.Int) { + self.data.Balance = amount + if self.onDirty != nil { + self.onDirty(self.Address()) + self.onDirty = nil + } } // Return the gas back to the origin. Used by the Virtual machine or Closures func (c *StateObject) ReturnGas(gas, price *big.Int) {} -func (self *StateObject) Copy(db trie.Database) *StateObject { - stateObject := NewObject(self.address, self.data) - stateObject.data.Balance.Set(self.data.Balance) +func (self *StateObject) Copy(db trie.Database, onDirty func(addr common.Address)) *StateObject { + stateObject := NewObject(self.address, self.data, onDirty) stateObject.trie = self.trie stateObject.code = self.code stateObject.storage = self.storage.Copy() stateObject.remove = self.remove - stateObject.dirty = self.dirty stateObject.dirtyCode = self.dirtyCode stateObject.deleted = self.deleted return stateObject @@ -282,12 +288,19 @@ func (self *StateObject) SetCode(code []byte) { self.data.CodeHash = crypto.Keccak256(code) self.data.codeSize = new(int) *self.data.codeSize = len(code) - self.dirty, self.dirtyCode = true, true + self.dirtyCode = true + if self.onDirty != nil { + self.onDirty(self.Address()) + self.onDirty = nil + } } func (self *StateObject) SetNonce(nonce uint64) { self.data.Nonce = nonce - self.dirty = true + if self.onDirty != nil { + self.onDirty(self.Address()) + self.onDirty = nil + } } func (self *StateObject) CodeHash() []byte { diff --git a/core/state/state_test.go b/core/state/state_test.go index c88274f3b2..fcdc385884 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -151,9 +151,10 @@ func TestSnapshot2(t *testing.T) { so0.SetCode([]byte{'c', 'a', 'f', 'e'}) so0.remove = false so0.deleted = false - so0.dirty = true state.SetStateObject(so0) - state.Commit() + + root, _ := state.Commit() + state.Reset(root) // and one with deleted == true so1 := state.GetStateObject(stateobjaddr1) @@ -162,7 +163,6 @@ func TestSnapshot2(t *testing.T) { so1.SetCode([]byte{'c', 'a', 'f', 'e', '2'}) so1.remove = true so1.deleted = true - so1.dirty = true state.SetStateObject(so1) so1 = state.GetStateObject(stateobjaddr1) @@ -183,7 +183,7 @@ func TestSnapshot2(t *testing.T) { // deleted should be nil, both before and after restore of state copy so1Restored := state.GetStateObject(stateobjaddr1) if so1Restored != nil { - t.Fatalf("deleted object not nil after restoring snapshot") + t.Fatalf("deleted object not nil after restoring snapshot: %+v", so1Restored) } } @@ -227,7 +227,4 @@ func compareStateObjects(so0, so1 *StateObject, t *testing.T) { if so0.deleted != so1.deleted { t.Fatalf("Deleted mismatch: have %v, want %v", so0.deleted, so1.deleted) } - if so0.dirty != so1.dirty { - t.Fatalf("Dirty mismatch: have %v, want %v", so0.dirty, so1.dirty) - } } diff --git a/core/state/statedb.go b/core/state/statedb.go index bcdc48d048..e1e0420902 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -54,9 +54,9 @@ type StateDB struct { accountBloom *boom.ScalableBloomFilter accountBloomDirty bool - // This map holds 'live' objects, which will get modified - // while processing a state transition. - stateObjects map[common.Address]*StateObject + // This map holds 'live' objects, which will get modified while processing a state transition. + stateObjects map[common.Address]*StateObject + stateObjectsDirty map[common.Address]struct{} // The refund counter, also used by state transitioning. refund *big.Int @@ -74,12 +74,13 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { return nil, err } return &StateDB{ - db: db, - trie: tr, - all: make(map[common.Address]Account), - stateObjects: make(map[common.Address]*StateObject), - refund: new(big.Int), - logs: make(map[common.Hash]vm.Logs), + db: db, + trie: tr, + all: make(map[common.Address]Account), + stateObjects: make(map[common.Address]*StateObject), + stateObjectsDirty: make(map[common.Address]struct{}), + refund: new(big.Int), + logs: make(map[common.Hash]vm.Logs), }, nil } @@ -99,13 +100,14 @@ func (self *StateDB) Reset(root common.Hash) error { all = make(map[common.Address]Account) } *self = StateDB{ - db: self.db, - trie: tr, - all: all, - accountBloom: self.accountBloom, - stateObjects: make(map[common.Address]*StateObject), - refund: new(big.Int), - logs: make(map[common.Hash]vm.Logs), + db: self.db, + trie: tr, + all: all, + accountBloom: self.accountBloom, + stateObjects: make(map[common.Address]*StateObject), + stateObjectsDirty: make(map[common.Address]struct{}), + refund: new(big.Int), + logs: make(map[common.Hash]vm.Logs), } return nil } @@ -267,7 +269,6 @@ func (self *StateDB) DeleteStateObject(stateObject *StateObject) { addr := stateObject.Address() self.trie.Delete(addr[:]) - //delete(self.stateObjects, addr) } // Retrieve a state object given my the address. Returns nil if not found. @@ -281,7 +282,7 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje } // Use cached account data from the canon state if possible. if data, ok := self.all[addr]; ok { - obj := NewObject(addr, data) + obj := NewObject(addr, data, self.MarkStateObjectDirty) self.SetStateObject(obj) return obj } @@ -304,7 +305,7 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje // The object we just loaded has no storage trie and code yet. self.all[addr] = data // Insert into the live set. - obj := NewObject(addr, data) + obj := NewObject(addr, data, self.MarkStateObjectDirty) self.SetStateObject(obj) return obj } @@ -328,9 +329,8 @@ func (self *StateDB) newStateObject(addr common.Address) *StateObject { if glog.V(logger.Core) { glog.Infof("(+) %x\n", addr) } - obj := NewObject(addr, Account{}) - obj.dirty = true - obj.SetNonce(StartingNonce) + obj := NewObject(addr, Account{}, self.MarkStateObjectDirty) + obj.SetNonce(StartingNonce) // sets the object to dirty self.stateObjects[addr] = obj if !self.getAccountBloom().TestAndAdd(self.trie.HashKey(addr[:])) { self.accountBloomDirty = true @@ -338,6 +338,12 @@ func (self *StateDB) newStateObject(addr common.Address) *StateObject { return obj } +// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly +// state object cache iteration to find a handful of modified ones. +func (self *StateDB) MarkStateObjectDirty(addr common.Address) { + self.stateObjectsDirty[addr] = struct{}{} +} + // Creates creates a new state object and takes ownership. func (self *StateDB) CreateStateObject(addr common.Address) *StateObject { // Get previous (if any) @@ -361,32 +367,35 @@ func (self *StateDB) CreateAccount(addr common.Address) vm.Account { // func (self *StateDB) Copy() *StateDB { - // ignore error - we assume state-to-be-copied always exists - state, _ := New(common.Hash{}, self.db) - state.trie = self.trie - state.all = self.all - state.accountBloom = self.accountBloom - state.accountBloomDirty = self.accountBloomDirty - for addr, stateObject := range self.stateObjects { - if stateObject.dirty { - state.stateObjects[addr] = stateObject.Copy(self.db) - } + // Copy all the basic fields, initialize the memory ones + state := &StateDB{ + db: self.db, + trie: self.trie, + all: self.all, + accountBloom: self.accountBloom, + accountBloomDirty: self.accountBloomDirty, + stateObjects: make(map[common.Address]*StateObject, len(self.stateObjectsDirty)), + stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)), + refund: new(big.Int).Set(self.refund), + logs: make(map[common.Hash]vm.Logs, len(self.logs)), + logSize: self.logSize, + } + // Copy the dirty states and logs + for addr, _ := range self.stateObjectsDirty { + state.stateObjects[addr] = self.stateObjects[addr].Copy(self.db, state.MarkStateObjectDirty) + state.stateObjectsDirty[addr] = struct{}{} } - - state.refund.Set(self.refund) - for hash, logs := range self.logs { state.logs[hash] = make(vm.Logs, len(logs)) copy(state.logs[hash], logs) } - state.logSize = self.logSize - return state } func (self *StateDB) Set(state *StateDB) { self.trie = state.trie self.stateObjects = state.stateObjects + self.stateObjectsDirty = state.stateObjectsDirty self.all = state.all self.accountBloom = state.accountBloom self.accountBloomDirty = state.accountBloomDirty @@ -405,14 +414,13 @@ func (self *StateDB) GetRefund() *big.Int { // goes into transaction receipts. func (s *StateDB) IntermediateRoot() common.Hash { s.refund = new(big.Int) - for _, stateObject := range s.stateObjects { - if stateObject.dirty { - if stateObject.remove { - s.DeleteStateObject(stateObject) - } else { - stateObject.UpdateRoot(s.db) - s.UpdateStateObject(stateObject) - } + for addr, _ := range s.stateObjectsDirty { + stateObject := s.stateObjects[addr] + if stateObject.remove { + s.DeleteStateObject(stateObject) + } else { + stateObject.UpdateRoot(s.db) + s.UpdateStateObject(stateObject) } } return s.trie.Hash() @@ -427,15 +435,15 @@ func (s *StateDB) DeleteSuicides() { // Reset refund so that any used-gas calculations can use // this method. s.refund = new(big.Int) - for _, stateObject := range s.stateObjects { - if stateObject.dirty { - // If the object has been removed by a suicide - // flag the object as deleted. - if stateObject.remove { - stateObject.deleted = true - } - stateObject.dirty = false + for addr, _ := range s.stateObjectsDirty { + stateObject := s.stateObjects[addr] + + // If the object has been removed by a suicide + // flag the object as deleted. + if stateObject.remove { + stateObject.deleted = true } + delete(s.stateObjectsDirty, addr) } } @@ -470,7 +478,7 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) // and just mark it for deletion in the trie. s.DeleteStateObject(stateObject) delete(s.all, addr) - } else if stateObject.dirty { + } else if _, ok := s.stateObjectsDirty[addr]; ok { // Write any contract code associated with the state object if stateObject.code != nil && stateObject.dirtyCode { if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil { @@ -486,7 +494,7 @@ func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) s.UpdateStateObject(stateObject) s.all[addr] = stateObject.data } - stateObject.dirty = false + delete(s.stateObjectsDirty, addr) } // Write account bloom and trie changes if s.accountBloomDirty { @@ -522,7 +530,7 @@ func (self *StateDB) getAccountBloom() *boom.ScalableBloomFilter { return self.accountBloom } // If the database does not yet contain a bloom filter, generate a new one - glog.V(logger.Info).Infof("generating accounts bloom, please wait...") + glog.V(logger.Info).Infof("generating account bloom, please wait...") filter := boom.NewDefaultScalableBloomFilter(0.01) start, pref := time.Now(), byte(0x00) @@ -531,13 +539,17 @@ func (self *StateDB) getAccountBloom() *boom.ScalableBloomFilter { for it.Next() { if pref != it.Key[0] { pref = it.Key[0] - glog.V(logger.Info).Infof("generating accounts bloom, at %.2f%%, please wait...", float64(pref)/2.56) + glog.V(logger.Info).Infof("generating account bloom, at %.2f%%, please wait...", float64(pref)/2.56) } filter.Add(it.Key) } - glog.V(logger.Info).Infof("accounts bloom generated in %v", time.Since(start)) + glog.V(logger.Info).Infof("account bloom generated in %v", time.Since(start)) - self.accountBloom, self.accountBloomDirty = filter, true + self.accountBloom = filter + if err := self.writeAccountsBloom(self.db); err != nil { + glog.V(logger.Error).Infof("failed to write initial account bloom filter: %v", err) + return nil + } return self.accountBloom } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 9283334597..12a8415205 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -17,6 +17,7 @@ package state import ( + "bytes" "math/big" "testing" @@ -47,6 +48,9 @@ func TestUpdateLeaks(t *testing.T) { // Ensure that no data was leaked into the database for _, key := range db.Keys() { value, _ := db.Get(key) + if bytes.Equal(key, []byte("accounts-bloom")) { + continue // This is written on first read + } t.Errorf("State leaked into database: %x -> %x", key, value) } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 725d1acea8..67e4bf832e 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -97,7 +97,7 @@ func benchStateTest(ruleSet RuleSet, test VmTest, env map[string]string, b *test db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) for addr, account := range test.Pre { - obj := StateObjectFromAccount(db, addr, account) + obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty) statedb.SetStateObject(obj) for a, v := range account.Storage { obj.SetState(common.HexToHash(a), common.HexToHash(v)) @@ -136,7 +136,7 @@ func runStateTest(ruleSet RuleSet, test VmTest) error { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) for addr, account := range test.Pre { - obj := StateObjectFromAccount(db, addr, account) + obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty) statedb.SetStateObject(obj) for a, v := range account.Storage { obj.SetState(common.HexToHash(a), common.HexToHash(v)) diff --git a/tests/util.go b/tests/util.go index 8cd07726c9..08fac2dd1b 100644 --- a/tests/util.go +++ b/tests/util.go @@ -103,7 +103,7 @@ func (self Log) Topics() [][]byte { return t } -func StateObjectFromAccount(db ethdb.Database, addr string, account Account) *state.StateObject { +func StateObjectFromAccount(db ethdb.Database, addr string, account Account, onDirty func(common.Address)) *state.StateObject { if common.IsHex(account.Code) { account.Code = account.Code[2:] } @@ -112,7 +112,7 @@ func StateObjectFromAccount(db ethdb.Database, addr string, account Account) *st Balance: common.Big(account.Balance), CodeHash: crypto.Keccak256(code), Nonce: common.Big(account.Nonce).Uint64(), - }) + }, onDirty) obj.SetCode(code) return obj } diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index 431fef5c41..4ad72d91c1 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -103,7 +103,7 @@ func benchVmTest(test VmTest, env map[string]string, b *testing.B) { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) for addr, account := range test.Pre { - obj := StateObjectFromAccount(db, addr, account) + obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty) statedb.SetStateObject(obj) for a, v := range account.Storage { obj.SetState(common.HexToHash(a), common.HexToHash(v)) @@ -154,7 +154,7 @@ func runVmTest(test VmTest) error { db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) for addr, account := range test.Pre { - obj := StateObjectFromAccount(db, addr, account) + obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty) statedb.SetStateObject(obj) for a, v := range account.Storage { obj.SetState(common.HexToHash(a), common.HexToHash(v))