core, core/vm: limited state writes and limited state copies

Limited state writes by tracking objects which have been changed. The
same principle is used during state forking by tracking which state fork
owns / changed what.

WIP: the name `ownedStateObjects` will change to `changedStateObjects`

cmd/evm/main.go

@@ -44,6 +44,10 @@ var (
 		Name:  "debug",
 		Usage: "output full trace logs",
 	}
+	VMStatsFlag = cli.BoolFlag{
+		Name:  "vmstats",
+		Usage: "Display EVM statistics",
+	}
 	CodeFlag = cli.StringFlag{
 		Name:  "code",
 		Usage: "EVM code",
@@ -113,14 +117,20 @@ var (
 		Name:  "create",
 		Usage: "indicates the action should be create rather than call",
 	}
+	NoOptimiseFlag = cli.BoolFlag{
+		Name:  "nooptimise",
+		Usage: "disable program optimisations",
+	}
 )
 
 func init() {
 	app.Flags = []cli.Flag{
 		CreateFlag,
+		NoOptimiseFlag,
 		DebugFlag,
 		VerbosityFlag,
 		SysStatFlag,
+		VMStatsFlag,
 		CodeFlag,
 		GasFlag,
 		PriceFlag,
@@ -172,8 +182,9 @@ func run(ctx *cli.Context) error {
 		backend,
 		ruleSet{},
 		vm.Config{
-			Debug:  ctx.GlobalBool(DebugFlag.Name),
-			Tracer: logger,
+			Debug:      ctx.GlobalBool(DebugFlag.Name),
+			Tracer:     logger,
+			NoOptimise: ctx.GlobalBool(NoOptimiseFlag.Name),
 		},
 	)
 
@@ -208,7 +219,10 @@ func run(ctx *cli.Context) error {
 		state.Commit(st)
 		fmt.Println(string(st.Dump()))
 	}
-	vm.StdErrFormat(logger.StructLogs())
+
+	if ctx.GlobalBool(DebugFlag.Name) {
+		vm.StdErrFormat(logger.StructLogs())
+	}
 
 	if ctx.GlobalBool(SysStatFlag.Name) {
 		var mem runtime.MemStats
@@ -223,7 +237,13 @@ num gc:     %d
 `, mem.Alloc, mem.TotalAlloc, mem.Mallocs, mem.HeapAlloc, mem.HeapObjects, mem.NumGC)
 	}
 
-	fmt.Printf("OUT: 0x%x", ret)
+	if ctx.GlobalBool(VMStatsFlag.Name) {
+		fmt.Printf(`VM Stats
+Gas requirement: %v
+`, vmenv.Gasser.HighwaterMark())
+	}
+
+	fmt.Printf("0x%x", ret)
 	if err != nil {
 		fmt.Printf(" error: %v", err)
 	}

core/blockchain.go

@@ -914,13 +914,16 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			reportBlock(block, err)
 			return i, err
 		}
+		tstart := time.Now()
 		// flatten the state before committing.
 		st = state.Flatten(st)
+		fmt.Println("reduce took", time.Since(tstart))
 		// Write state changes to database
 		_, err = state.Commit(st)
 		if err != nil {
 			return i, err
 		}
+		glog.V(logger.Info).Infoln("state stats:", st)
 
 		// coalesce logs for later processing
 		coalescedLogs = append(coalescedLogs, logs...)
@@ -938,7 +941,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 
 		switch status {
 		case CanonStatTy:
-			if glog.V(logger.Debug) {
+			if glog.V(logger.Info) {
 				glog.Infof("[%v] inserted block #%d (%d TXs %v G %d UNCs) (%x...). Took %v\n", time.Now().UnixNano(), block.Number(), len(block.Transactions()), block.GasUsed(), len(block.Uncles()), block.Hash().Bytes()[0:4], time.Since(bstart))
 			}
 			events = append(events, ChainEvent{block, block.Hash(), logs})

core/execution.go

@@ -69,13 +69,13 @@ func (c EVMCallContext) exec(env *vm.Environment, caller vm.ContractRef, address
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
 	if env.Depth > int(params.CallCreateDepth.Int64()) {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, common.Address{}, vm.DepthError
 	}
 
 	if !c.CanTransfer(env.Db(), caller.Address(), value) {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, common.Address{}, ValueTransferErr("insufficient funds to transfer value. Req %v, has %v", value, env.Db().GetBalance(caller.Address()))
 	}
@@ -113,7 +113,9 @@ func (c EVMCallContext) exec(env *vm.Environment, caller vm.ContractRef, address
 	// only.
 	contract := vm.NewContract(caller, to, value, gas)
 	contract.SetCallCode(codeAddr, code)
-	defer contract.Finalise()
+	defer func() {
+		contract.Finalise(env.Gasser)
+	}()
 
 	ret, err = evm.Run(contract, input)
 	// if the contract creation ran successfully and no errors were returned
@@ -123,7 +125,7 @@ func (c EVMCallContext) exec(env *vm.Environment, caller vm.ContractRef, address
 	if err == nil && createAccount {
 		dataGas := big.NewInt(int64(len(ret)))
 		dataGas.Mul(dataGas, params.CreateDataGas)
-		if contract.UseGas(dataGas.Uint64()) {
+		if env.Gasser.UseGas(contract, dataGas.Uint64()) { //contract.UseGas(dataGas.Uint64()) {
 			env.Db().SetCode(*address, ret)
 		} else {
 			err = vm.CodeStoreOutOfGasError
@@ -134,7 +136,7 @@ func (c EVMCallContext) exec(env *vm.Environment, caller vm.ContractRef, address
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in homestead this also counts for code storage gas errors.
 	if err != nil && (env.RuleSet().IsHomestead(env.BlockNumber) || err != vm.CodeStoreOutOfGasError) {
-		contract.UseGas(contract.Gas())
+		env.Gasser.UseGas(contract, contract.Gas()) //contract.UseGas(contract.Gas())
 
 		env.Backend.Set(snapshotPreTransfer)
 	}
@@ -147,7 +149,8 @@ func (c EVMCallContext) execDelegateCall(env *vm.Environment, caller vm.Contract
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
 	if env.Depth > int(params.CallCreateDepth.Int64()) {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
+
 		return nil, common.Address{}, vm.DepthError
 	}
 
@@ -164,11 +167,13 @@ func (c EVMCallContext) execDelegateCall(env *vm.Environment, caller vm.Contract
 	// Iinitialise a new contract and make initialise the delegate values
 	contract := vm.NewContract(caller, to, value, gas).AsDelegate()
 	contract.SetCallCode(codeAddr, code)
-	defer contract.Finalise()
+	defer func() {
+		contract.Finalise(env.Gasser)
+	}()
 
 	ret, err = evm.Run(contract, input)
 	if err != nil {
-		contract.UseGas(contract.Gas())
+		env.Gasser.UseGas(contract, contract.Gas()) //contract.UseGas(contract.Gas())
 
 		env.Backend.Set(snapshot)
 	}

core/state/state.go

@@ -21,8 +21,9 @@ type State struct {
 
 	parent *State
 
-	StateObjects map[common.Address]*StateObject
-	refund       *big.Int
+	StateObjects      map[common.Address]*StateObject
+	ownedStateObjects map[common.Address]bool
+	refund            *big.Int
 
 	logIdx uint
 	logs   []*vm.Log
@@ -44,10 +45,11 @@ func New(root common.Hash, db ethdb.Database) (*State, error) {
 	}
 
 	return &State{
-		Db:           db,
-		Trie:         tr,
-		StateObjects: make(map[common.Address]*StateObject),
-		refund:       new(big.Int),
+		Db:                db,
+		Trie:              tr,
+		StateObjects:      make(map[common.Address]*StateObject),
+		ownedStateObjects: make(map[common.Address]bool),
+		refund:            new(big.Int),
 	}, nil
 }
 
@@ -90,9 +92,32 @@ func (s *State) Read(address common.Address) (object *StateObject, inCache bool)
 		glog.Errorf("can't decode object at %x: %v", address[:], err)
 		return nil, false
 	}
+
 	return stateObject, false
 }
 
+func (s *State) GetOrNewStateObject(address common.Address) *StateObject {
+	stateObject, inCache := s.Read(address)
+	if stateObject != nil {
+		if !inCache || !s.ownedStateObjects[address] {
+			stateObject = stateObject.Copy()
+			s.StateObjects[address] = stateObject
+			s.ownedStateObjects[address] = true
+		}
+		return stateObject
+	}
+
+	if stateObject == nil || stateObject.deleted {
+		stateObject = NewStateObject(address, s.Db)
+		stateObject.SetNonce(StartingNonce)
+
+		s.StateObjects[address] = stateObject
+		s.ownedStateObjects[address] = true
+	}
+	return stateObject
+}
+
+/*
 func (s *State) GetOrNewStateObject(address common.Address) *StateObject {
 	stateObject := s.GetStateObject(address)
 	if stateObject == nil || stateObject.deleted {
@@ -104,14 +129,15 @@ func (s *State) GetOrNewStateObject(address common.Address) *StateObject {
 
 	return stateObject
 }
+*/
 
 func (s *State) GetStateObject(address common.Address) *StateObject {
 	account, inCache := s.Read(address)
 	if account != nil {
 		if !inCache {
-			s.StateObjects[address] = account.Copy()
+			s.StateObjects[address] = account
 		}
-		return s.StateObjects[address]
+		return account
 	}
 	return nil
 }
@@ -294,6 +320,7 @@ func (s *State) UpdateStateObject(stateObject *StateObject) {
 }
 
 func (s *State) Reset(root common.Hash) error {
+	fmt.Println("reset")
 	var (
 		err error
 		tr  = s.Trie
@@ -304,10 +331,13 @@ func (s *State) Reset(root common.Hash) error {
 		}
 	}
 	*s = State{
-		Db:           s.Db,
-		Trie:         tr,
-		StateObjects: make(map[common.Address]*StateObject),
-		refund:       new(big.Int),
+		Db:                s.Db,
+		Trie:              tr,
+		StateObjects:      s.StateObjects,
+		ownedStateObjects: s.ownedStateObjects,
+		//StateObjects:      make(map[common.Address]*StateObject),
+		//ownedStateObjects: make(map[common.Address]bool),
+		refund: new(big.Int),
 	}
 	return nil
 }
@@ -345,11 +375,12 @@ func Fork(parent *State) *State {
 		Db:   parent.Db,
 		Trie: parent.Trie,
 
-		parent:       parent,
-		StateObjects: make(map[common.Address]*StateObject),
-		refund:       new(big.Int),
-		logIdx:       parent.logIdx,
-		logs:         nil,
+		parent:            parent,
+		StateObjects:      make(map[common.Address]*StateObject),
+		ownedStateObjects: make(map[common.Address]bool),
+		refund:            new(big.Int),
+		logIdx:            parent.logIdx,
+		logs:              nil,
 	}
 }
 
@@ -366,15 +397,19 @@ func Flatten(s *State) *State {
 		flattenedState = Flatten(s.parent)
 	} else {
 		flattenedState = &State{
-			Db:           s.Db,
-			Trie:         s.Trie,
-			refund:       new(big.Int),
-			StateObjects: make(map[common.Address]*StateObject),
+			Db:                s.Db,
+			Trie:              s.Trie,
+			refund:            new(big.Int),
+			StateObjects:      make(map[common.Address]*StateObject),
+			ownedStateObjects: make(map[common.Address]bool),
 		}
 	}
 
 	for address, object := range s.StateObjects {
 		flattenedState.StateObjects[address] = object
+		if s.ownedStateObjects[address] {
+			flattenedState.ownedStateObjects[address] = true
+		}
 	}
 
 	flattenedState.logs = append(flattenedState.logs, s.logs...)
@@ -383,6 +418,10 @@ func Flatten(s *State) *State {
 	return flattenedState
 }
 
+func (s *State) String() string {
+	return fmt.Sprintf("objects: %d owned: %d", len(s.StateObjects), len(s.ownedStateObjects))
+}
+
 func IntermediateRoot(state *State) common.Hash {
 	s := Flatten(state)
 

core/state/state_write.go

@@ -27,13 +27,13 @@ func CommitBatch(state *State) (common.Hash, ethdb.Batch) {
 func stateCommit(state *State, db trie.DatabaseWriter) (common.Hash, error) {
 	// make sure the state is flattened before committing
 	state = Flatten(state)
-	for _, stateObject := range state.StateObjects {
+	for address, stateObject := range state.StateObjects {
 		if stateObject.remove {
 			// If the object has been removed, don't bother syncing it
 			// and just mark it for deletion in the trie.
 			stateObject.deleted = true
 			state.Trie.Delete(stateObject.Address().Bytes()[:])
-		} else {
+		} else if state.ownedStateObjects[address] {
 			// Write any contract code associated with the state object
 			if len(stateObject.code) > 0 {
 				if err := db.Put(stateObject.codeHash, stateObject.code); err != nil {

core/vm/contract.go

@@ -114,14 +114,14 @@ func (c *Contract) Caller() common.Address {
 
 // Finalise finalises the contract and returning any remaining gas to the original
 // caller.
-func (c *Contract) Finalise() {
+func (c *Contract) Finalise(gasser *Gasser) {
 	c.gas.SetUint64(c.gas64)
 	// Return the remaining gas to the caller
-	c.caller.ReturnGas(c.gas64)
+	gasser.ReturnGas(c.caller, c.gas64)
 }
 
 // UseGas attempts the use gas and subtracts it and returns true on success
-func (c *Contract) UseGas(gas uint64) (ok bool) {
+func (c *Contract) useGas(gas uint64) (ok bool) {
 	if c.gas64 < gas {
 		return false
 	}

core/vm/contracts.go

@@ -45,7 +45,7 @@ var PrecompiledContracts = map[common.Address]PrecompiledContract{
 // RunPrecompile runs and evaluate the output of a precompiled contract defined in contracts.go
 func RunPrecompiled(p PrecompiledContract, input []byte, contract *Contract) (ret []byte, err error) {
 	gas := p.RequiredGas(len(input))
-	if contract.UseGas(gas.Uint64()) {
+	if contract.useGas(gas.Uint64()) {
 		ret = p.Run(input)
 
 		return ret, nil

core/vm/instructions.go

@@ -69,9 +69,14 @@ func (instr instruction) do(program *Program, pc *uint64, env *Environment, cont
 
 	// Use the calculated gas. When insufficient gas is present, use all gas and return an
 	// Out Of Gas error
-	if !contract.UseGas(cost) {
+	if !env.Gasser.UseGas(contract, cost) {
 		return nil, OutOfGasError
 	}
+	/*
+		if !contract.UseGas(cost) {
+			return nil, OutOfGasError
+		}
+	*/
 	// Resize the memory calculated previously
 	memory.Resize(newMemSize)
 
@@ -515,10 +520,11 @@ func opCreate(instr instruction, pc *uint64, env *Environment, contract *Contrac
 		value        = stack.pop()
 		offset, size = stack.pop(), stack.pop()
 		input        = memory.Get(offset.Int64(), size.Int64())
-		gas          = new(big.Int).SetUint64(contract.gas64)
+		gas          = contract.gas64
 	)
-	contract.UseGas(contract.gas64)
-	_, addr, suberr := env.Create(contract, input, gas, value)
+
+	contract.useGas(gas)
+	_, addr, suberr := env.Create(contract, input, new(big.Int).SetUint64(gas), value)
 	// Push item on the stack based on the returned error. If the ruleset is
 	// homestead we must check for CodeStoreOutOfGasError (homestead only
 	// rule) and treat as an error, if the ruleset is frontier we must
@@ -530,6 +536,8 @@ func opCreate(instr instruction, pc *uint64, env *Environment, contract *Contrac
 	} else {
 		stack.push(addr.Big())
 	}
+
+	env.Gasser.mark(gas - contract.Gas())
 }
 
 func opCall(instr instruction, pc *uint64, env *Environment, contract *Contract, memory *Memory, stack *Stack) {

core/vm/interface.go

@@ -78,6 +78,36 @@ type Account interface {
 	Value() *big.Int
 }
 
+type Gasser struct {
+	highwaterMark uint64
+	slider        uint64
+}
+
+func (g *Gasser) mark(gas uint64) {
+	g.slider += gas
+	if g.slider > g.highwaterMark {
+		g.highwaterMark = g.slider
+	}
+}
+
+func (g *Gasser) UseGas(contract *Contract, gas uint64) bool {
+	ok := contract.useGas(gas)
+
+	if ok {
+		g.mark(gas)
+	}
+	return ok
+}
+
+func (g *Gasser) ReturnGas(contract ContractRef, gas uint64) {
+	g.slider -= gas
+	contract.ReturnGas(gas)
+}
+
+func (g *Gasser) HighwaterMark() uint64 {
+	return g.highwaterMark
+}
+
 // Context provides the EVM with auxilary information. Once provided it shouldn't be modified.
 type Context struct {
 	CallContext
@@ -121,6 +151,7 @@ type Environment struct {
 	Context
 
 	Backend Backend
+	Gasser  *Gasser
 
 	ruleSet  RuleSet
 	vmConfig Config
@@ -133,6 +164,7 @@ type Environment struct {
 func NewEnvironment(context Context, backend Backend, ruleSet RuleSet, vmCfg Config) *Environment {
 	env := &Environment{
 		Context:  context,
+		Gasser:   new(Gasser),
 		Backend:  backend,
 		vmConfig: vmCfg,
 		ruleSet:  ruleSet,
@@ -143,7 +175,7 @@ func NewEnvironment(context Context, backend Backend, ruleSet RuleSet, vmCfg Con
 
 func (env *Environment) Call(caller ContractRef, addr common.Address, data []byte, gas, value *big.Int) ([]byte, error) {
 	if env.vmConfig.Test && env.Depth > 0 {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, nil
 	}
@@ -154,7 +186,7 @@ func (env *Environment) Call(caller ContractRef, addr common.Address, data []byt
 // Take another's contract code and execute within our own context
 func (env *Environment) CallCode(caller ContractRef, addr common.Address, data []byte, gas, value *big.Int) ([]byte, error) {
 	if env.vmConfig.Test && env.Depth > 0 {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, nil
 	}
@@ -165,7 +197,7 @@ func (env *Environment) CallCode(caller ContractRef, addr common.Address, data [
 // Same as CallCode except sender and value is propagated from parent to child scope
 func (env *Environment) DelegateCall(caller ContractRef, addr common.Address, data []byte, gas *big.Int) ([]byte, error) {
 	if env.vmConfig.Test && env.Depth > 0 {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, nil
 	}
@@ -176,7 +208,7 @@ func (env *Environment) DelegateCall(caller ContractRef, addr common.Address, da
 // Create a new contract
 func (env *Environment) Create(caller ContractRef, data []byte, gas, value *big.Int) ([]byte, common.Address, error) {
 	if env.vmConfig.Test && env.Depth > 0 {
-		caller.ReturnGas(gas.Uint64())
+		env.Gasser.ReturnGas(caller, gas.Uint64())
 
 		return nil, common.Address{}, nil
 	}

core/vm/segments.go

@@ -25,7 +25,7 @@ type jumpSeg struct {
 }
 
 func (j jumpSeg) do(program *Program, pc *uint64, env *Environment, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	if !contract.UseGas(j.gas) {
+	if !env.Gasser.UseGas(contract, j.gas) { //!contract.UseGas(j.gas) {
 		return nil, OutOfGasError
 	}
 	if j.err != nil {
@@ -45,7 +45,7 @@ type pushSeg struct {
 func (s pushSeg) do(program *Program, pc *uint64, env *Environment, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	// Use the calculated gas. When insufficient gas is present, use all gas and return an
 	// Out Of Gas error
-	if !contract.UseGas(s.gas) {
+	if !env.Gasser.UseGas(contract, s.gas) { //!contract.UseGas(s.gas) {
 		return nil, OutOfGasError
 	}
 

core/vm/vm.go

@@ -61,7 +61,13 @@ func (evm *EVM) Run(contract *Contract, input []byte) ([]byte, error) {
 
 	if contract.CodeAddr != nil {
 		if p, exist := PrecompiledContracts[*contract.CodeAddr]; exist {
-			return RunPrecompiled(p, input, contract)
+			gas := p.RequiredGas(len(input))
+			if evm.env.Gasser.UseGas(contract, gas.Uint64()) { //contract.UseGas(gas.Uint64()) {
+				return p.Run(input), nil
+			} else {
+				return nil, OutOfGasError
+			}
+			//return RunPrecompiled(p, input, contract)
 		}
 	}
 

internal/ethapi/api.go

@@ -1280,8 +1280,8 @@ func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) {
 }
 
 // SetHead rewinds the head of the blockchain to a previous block.
-func (api *PrivateDebugAPI) SetHead(number uint64) {
-	api.b.SetHead(number)
+func (api *PrivateDebugAPI) SetHead(number string) {
+	api.b.SetHead(common.String2Big(number).Uint64())
 }
 
 // PublicNetAPI offers network related RPC methods