core, core/state: WIP cache all teh things

cmd/geth/main.go

@@ -136,6 +136,7 @@ participating.
 		utils.OlympicFlag,
 		utils.FastSyncFlag,
 		utils.CacheFlag,
+		utils.StateCacheFlag,
 		utils.LightKDFFlag,
 		utils.JSpathFlag,
 		utils.ListenPortFlag,

cmd/geth/usage.go

@@ -74,6 +74,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.FastSyncFlag,
 			utils.LightKDFFlag,
 			utils.CacheFlag,
+			utils.StateCacheFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -144,6 +144,10 @@ var (
 		Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)",
 		Value: 128,
 	}
+	StateCacheFlag = cli.BoolFlag{
+		Name:  "statecache",
+		Usage: "Caches the last known state before syncing with the network. Uses significantly more memory.",
+	}
 	FastSyncFlag = cli.BoolFlag{
 		Name:  "fast",
 		Usage: "Enable fast syncing through state downloads",

core/blockchain.go

@@ -107,6 +107,8 @@ type BlockChain struct {
 	pow       pow.PoW
 	processor Processor // block processor interface
 	validator Validator // block and state validator interface
+
+	canonState *state.State
 }
 
 // NewBlockChain returns a fully initialised block chain using information
@@ -205,19 +207,23 @@ func (self *BlockChain) loadLastState() error {
 	glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd)
 	glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd)
 
-	/*
+	return nil
-		st, err := state.New(currentHeader.Root, self.chainDb)
+}
-		if err != nil {
+
-			return err
+func (bc *BlockChain) PreloadState() error {
-		}
+	currentHeader := bc.currentBlock.Header()
-		glog.V(logger.Info).Infoln("caching state accounts...")
+
-		tstart := time.Now()
+	st, err := state.New(currentHeader.Root, bc.chainDb)
-		if err := st.LoadAll(); err != nil {
+	if err != nil {
-			return fmt.Errorf("core: could not pre-load account data: %v", err)
+		return err
-		}
+	}
-		self.canonState = st
+	glog.V(logger.Info).Infoln("caching state accounts...")
-		glog.V(logger.Info).Infoln("cached", len(st.StateObjects), "in", time.Since(tstart))
+	tstart := time.Now()
-	*/
+	if err := st.LoadAll(); err != nil {
+		return fmt.Errorf("core: could not pre-load account data: %v", err)
+	}
+	bc.canonState = st
+	glog.V(logger.Info).Infoln("cached", len(st.StateObjects), "in", time.Since(tstart))
 
 	return nil
 }
@@ -905,12 +911,17 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			return i, err
 		}
 
+		parent := self.GetBlock(block.ParentHash(), block.NumberU64()-1)
 		// Create a new statedb using the parent block and report an
 		// error if it fails.
 		if st == nil {
-			st, err = state.New(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root(), self.chainDb)
+			if self.canonState != nil && parent.Root() == common.BytesToHash(self.canonState.Trie.Root()) {
+				st = self.canonState
+			} else {
+				st, err = state.New(parent.Root(), self.chainDb)
+			}
 		} else {
-			err = st.Reset(chain[i-1].Root())
+			err = st.Reset(parent.Root())
 		}
 		if err != nil {
 			reportBlock(block, err)
@@ -928,10 +939,8 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			reportBlock(block, err)
 			return i, err
 		}
-		tstart := time.Now()
 		// flatten the state before committing.
 		st = state.Flatten(st)
-		fmt.Println("reduce took", time.Since(tstart))
 		// Write state changes to database
 		_, err = state.Commit(st)
 		if err != nil {
@@ -972,6 +981,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
 				return i, err
 			}
+			self.canonState = st
 		case SideStatTy:
 			if glog.V(logger.Detail) {
 				glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) (%x...). Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Bytes()[0:4], time.Since(bstart))

core/state/state.go

@@ -23,7 +23,7 @@ type State struct {
 	parent *State
 
 	StateObjects      map[common.Address]*StateObject
-	ownedStateObjects map[common.Address]bool
+	localStateObjects map[common.Address]bool
 	refund            *big.Int
 
 	logIdx uint
@@ -49,7 +49,7 @@ func New(root common.Hash, db ethdb.Database) (*State, error) {
 		Db:                db,
 		Trie:              tr,
 		StateObjects:      make(map[common.Address]*StateObject),
-		ownedStateObjects: make(map[common.Address]bool),
+		localStateObjects: make(map[common.Address]bool),
 		refund:            new(big.Int),
 	}, nil
 }
@@ -128,10 +128,10 @@ func (s *State) loadStateObject(stateObject *StateObject) {
 func (s *State) GetOrNewStateObject(address common.Address) *StateObject {
 	stateObject, inCache := s.Read(address)
 	if stateObject != nil {
-		if !inCache || !s.ownedStateObjects[address] {
+		if !inCache || !s.localStateObjects[address] {
 			stateObject = stateObject.Copy()
 			s.StateObjects[address] = stateObject
-			s.ownedStateObjects[address] = true
+			s.localStateObjects[address] = true
 		}
 		return stateObject
 	}
@@ -141,7 +141,7 @@ func (s *State) GetOrNewStateObject(address common.Address) *StateObject {
 		stateObject.SetNonce(StartingNonce)
 
 		s.StateObjects[address] = stateObject
-		s.ownedStateObjects[address] = true
+		s.localStateObjects[address] = true
 	}
 	return stateObject
 }
@@ -372,9 +372,8 @@ func (s *State) Reset(root common.Hash) error {
 		Db:                s.Db,
 		Trie:              tr,
 		StateObjects:      s.StateObjects,
-		ownedStateObjects: s.ownedStateObjects,
+		localStateObjects: make(map[common.Address]bool),
 		//StateObjects:      make(map[common.Address]*StateObject),
-		//ownedStateObjects: make(map[common.Address]bool),
 		refund: new(big.Int),
 	}
 	return nil
@@ -415,7 +414,7 @@ func Fork(parent *State) *State {
 
 		parent:            parent,
 		StateObjects:      make(map[common.Address]*StateObject),
-		ownedStateObjects: make(map[common.Address]bool),
+		localStateObjects: make(map[common.Address]bool),
 		refund:            new(big.Int),
 		logIdx:            parent.logIdx,
 		logs:              nil,
@@ -435,8 +434,8 @@ func Flatten(s *State) *State {
 
 	for address, object := range s.StateObjects {
 		flattenedState.StateObjects[address] = object
-		if s.ownedStateObjects[address] {
+		if s.localStateObjects[address] {
-			flattenedState.ownedStateObjects[address] = true
+			flattenedState.localStateObjects[address] = true
 		}
 	}
 
@@ -447,7 +446,7 @@ func Flatten(s *State) *State {
 }
 
 func (s *State) String() string {
-	return fmt.Sprintf("objects: %d owned: %d", len(s.StateObjects), len(s.ownedStateObjects))
+	return fmt.Sprintf("objects: %d owned: %d", len(s.StateObjects), len(s.localStateObjects))
 }
 
 func IntermediateRoot(state *State) common.Hash {

core/state/state_write.go

@@ -27,7 +27,9 @@ func CommitBatch(state *State) (common.Hash, ethdb.Batch) {
 func stateCommit(state *State, db trie.DatabaseWriter) (common.Hash, error) {
 	// make sure the state is flattened before committing
 	state = Flatten(state)
-	for _, stateObject := range state.StateObjects {
+	for address, _ := range state.localStateObjects {
+		stateObject := state.StateObjects[address]
+		//for _, stateObject := range state.StateObjects {
 		if stateObject.remove {
 			// If the object has been removed, don't bother syncing it
 			// and just mark it for deletion in the trie.