Merge branch 'swarm-network-rewrite' into swarm-network-rewrite-syncer

.github/CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Before you do a feature request please check and make sure that it isn't possible
 through some other means. The JavaScript enabled console is a powerful feature
-in the right hands. Please check our [Bitchin' tricks](https://github.com/ethereum/go-ethereum/wiki/bitchin-tricks) wiki page for more info
+in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
 ## Contributing

accounts/abi/abi.go

@@ -97,7 +97,6 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 		Type      string
 		Name      string
 		Constant  bool
-		Indexed   bool
 		Anonymous bool
 		Inputs    []Argument
 		Outputs   []Argument

accounts/abi/bind/backend.go

@@ -52,12 +52,6 @@ type ContractCaller interface {
 	CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error)
 }
 
-// DeployBackend wraps the operations needed by WaitMined and WaitDeployed.
-type DeployBackend interface {
-	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
-	CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error)
-}
-
 // PendingContractCaller defines methods to perform contract calls on the pending state.
 // Call will try to discover this interface when access to the pending state is requested.
 // If the backend does not support the pending state, Call returns ErrNoPendingState.
@@ -90,8 +84,29 @@ type ContractTransactor interface {
 	SendTransaction(ctx context.Context, tx *types.Transaction) error
 }
 
+// ContractFilterer defines the methods needed to access log events using one-off
+// queries or continuous event subscriptions.
+type ContractFilterer interface {
+	// FilterLogs executes a log filter operation, blocking during execution and
+	// returning all the results in one batch.
+	//
+	// TODO(karalabe): Deprecate when the subscription one can return past data too.
+	FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error)
+
+	// SubscribeFilterLogs creates a background log filtering operation, returning
+	// a subscription immediately, which can be used to stream the found events.
+	SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error)
+}
+
+// DeployBackend wraps the operations needed by WaitMined and WaitDeployed.
+type DeployBackend interface {
+	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
+	CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error)
+}
+
 // ContractBackend defines the methods needed to work with contracts on a read-write basis.
 type ContractBackend interface {
 	ContractCaller
 	ContractTransactor
+	ContractFilterer
 }

accounts/abi/bind/backends/simulated.go

@@ -30,11 +30,15 @@ import (
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
 )
 
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
@@ -53,6 +57,8 @@ type SimulatedBackend struct {
 	pendingBlock *types.Block   // Currently pending block that will be imported on request
 	pendingState *state.StateDB // Currently pending state that will be the active on on request
 
+	events *filters.EventSystem // Event system for filtering log events live
+
 	config *params.ChainConfig
 }
 
@@ -62,8 +68,14 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
-	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
+
+	backend := &SimulatedBackend{
+		database:   database,
+		blockchain: blockchain,
+		config:     genesis.Config,
+		events:     filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain}, false),
+	}
 	backend.rollback()
 	return backend
 }
@@ -90,8 +102,10 @@ func (b *SimulatedBackend) Rollback() {
 
 func (b *SimulatedBackend) rollback() {
 	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
+	statedb, _ := b.blockchain.State()
+
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
 }
 
 // CodeAt returns the code associated with a certain account in the blockchain.
@@ -248,7 +262,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 	return hi, nil
 }
 
-// callContract implemens common code between normal and pending contract calls.
+// callContract implements common code between normal and pending contract calls.
 // state is modified during execution, make sure to copy it if necessary.
 func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, block *types.Block, statedb *state.StateDB) ([]byte, uint64, bool, error) {
 	// Ensure message is initialized properly.
@@ -297,12 +311,76 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		}
 		block.AddTx(tx)
 	})
+	statedb, _ := b.blockchain.State()
+
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
 	return nil
 }
 
-// JumpTimeInSeconds adds skip seconds to the clock
+// FilterLogs executes a log filter operation, blocking during execution and
+// returning all the results in one batch.
+//
+// TODO(karalabe): Deprecate when the subscription one can return past data too.
+func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) {
+	// Initialize unset filter boundaried to run from genesis to chain head
+	from := int64(0)
+	if query.FromBlock != nil {
+		from = query.FromBlock.Int64()
+	}
+	to := int64(-1)
+	if query.ToBlock != nil {
+		to = query.ToBlock.Int64()
+	}
+	// Construct and execute the filter
+	filter := filters.New(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
+
+	logs, err := filter.Logs(ctx)
+	if err != nil {
+		return nil, err
+	}
+	res := make([]types.Log, len(logs))
+	for i, log := range logs {
+		res[i] = *log
+	}
+	return res, nil
+}
+
+// SubscribeFilterLogs creates a background log filtering operation, returning a
+// subscription immediately, which can be used to stream the found events.
+func (b *SimulatedBackend) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
+	// Subscribe to contract events
+	sink := make(chan []*types.Log)
+
+	sub, err := b.events.SubscribeLogs(query, sink)
+	if err != nil {
+		return nil, err
+	}
+	// Since we're getting logs in batches, we need to flatten them into a plain stream
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case logs := <-sink:
+				for _, log := range logs {
+					select {
+					case ch <- *log:
+					case err := <-sub.Err():
+						return err
+					case <-quit:
+						return nil
+					}
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// AdjustTime adds a time shift to the simulated clock.
 func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -312,8 +390,10 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 		}
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
+	statedb, _ := b.blockchain.State()
+
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
 
 	return nil
 }
@@ -331,3 +411,44 @@ func (m callmsg) GasPrice() *big.Int   { return m.CallMsg.GasPrice }
 func (m callmsg) Gas() uint64          { return m.CallMsg.Gas }
 func (m callmsg) Value() *big.Int      { return m.CallMsg.Value }
 func (m callmsg) Data() []byte         { return m.CallMsg.Data }
+
+// filterBackend implements filters.Backend to support filtering for logs without
+// taking bloom-bits acceleration structures into account.
+type filterBackend struct {
+	db ethdb.Database
+	bc *core.BlockChain
+}
+
+func (fb *filterBackend) ChainDb() ethdb.Database  { return fb.db }
+func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") }
+
+func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) {
+	if block == rpc.LatestBlockNumber {
+		return fb.bc.CurrentHeader(), nil
+	}
+	return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
+}
+func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
+	return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil
+}
+
+func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription {
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		<-quit
+		return nil
+	})
+}
+func (fb *filterBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	return fb.bc.SubscribeChainEvent(ch)
+}
+func (fb *filterBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+	return fb.bc.SubscribeRemovedLogsEvent(ch)
+}
+func (fb *filterBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	return fb.bc.SubscribeLogsEvent(ch)
+}
+
+func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 }
+func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) {
+	panic("not supported")
+}

accounts/abi/bind/base.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
 )
 
 // SignerFn is a signer function callback when a contract requires a method to
@@ -55,6 +56,22 @@ type TransactOpts struct {
 	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
 
+// FilterOpts is the collection of options to fine tune filtering for events
+// within a bound contract.
+type FilterOpts struct {
+	Start uint64  // Start of the queried range
+	End   *uint64 // End of the range (nil = latest)
+
+	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+}
+
+// WatchOpts is the collection of options to fine tune subscribing for events
+// within a bound contract.
+type WatchOpts struct {
+	Start   *uint64         // Start of the queried range (nil = latest)
+	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+}
+
 // BoundContract is the base wrapper object that reflects a contract on the
 // Ethereum network. It contains a collection of methods that are used by the
 // higher level contract bindings to operate.
@@ -63,16 +80,18 @@ type BoundContract struct {
 	abi        abi.ABI            // Reflect based ABI to access the correct Ethereum methods
 	caller     ContractCaller     // Read interface to interact with the blockchain
 	transactor ContractTransactor // Write interface to interact with the blockchain
+	filterer   ContractFilterer   // Event filtering to interact with the blockchain
 }
 
 // NewBoundContract creates a low level contract interface through which calls
 // and transactions may be made through.
-func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller, transactor ContractTransactor) *BoundContract {
+func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller, transactor ContractTransactor, filterer ContractFilterer) *BoundContract {
 	return &BoundContract{
 		address:    address,
 		abi:        abi,
 		caller:     caller,
 		transactor: transactor,
+		filterer:   filterer,
 	}
 }
 
@@ -80,7 +99,7 @@ func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller
 // deployment address with a Go wrapper.
 func DeployContract(opts *TransactOpts, abi abi.ABI, bytecode []byte, backend ContractBackend, params ...interface{}) (common.Address, *types.Transaction, *BoundContract, error) {
 	// Otherwise try to deploy the contract
-	c := NewBoundContract(common.Address{}, abi, backend, backend)
+	c := NewBoundContract(common.Address{}, abi, backend, backend, backend)
 
 	input, err := c.abi.Pack("", params...)
 	if err != nil {
@@ -225,6 +244,104 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	return signedTx, nil
 }
 
+// FilterLogs filters contract logs for past blocks, returning the necessary
+// channels to construct a strongly typed bound iterator on top of them.
+func (c *BoundContract) FilterLogs(opts *FilterOpts, name string, query ...[]interface{}) (chan types.Log, event.Subscription, error) {
+	// Don't crash on a lazy user
+	if opts == nil {
+		opts = new(FilterOpts)
+	}
+	// Append the event selector to the query parameters and construct the topic set
+	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+
+	topics, err := makeTopics(query...)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Start the background filtering
+	logs := make(chan types.Log, 128)
+
+	config := ethereum.FilterQuery{
+		Addresses: []common.Address{c.address},
+		Topics:    topics,
+		FromBlock: new(big.Int).SetUint64(opts.Start),
+	}
+	if opts.End != nil {
+		config.ToBlock = new(big.Int).SetUint64(*opts.End)
+	}
+	/* TODO(karalabe): Replace the rest of the method below with this when supported
+	sub, err := c.filterer.SubscribeFilterLogs(ensureContext(opts.Context), config, logs)
+	*/
+	buff, err := c.filterer.FilterLogs(ensureContext(opts.Context), config)
+	if err != nil {
+		return nil, nil, err
+	}
+	sub, err := event.NewSubscription(func(quit <-chan struct{}) error {
+		for _, log := range buff {
+			select {
+			case logs <- log:
+			case <-quit:
+				return nil
+			}
+		}
+		return nil
+	}), nil
+
+	if err != nil {
+		return nil, nil, err
+	}
+	return logs, sub, nil
+}
+
+// WatchLogs filters subscribes to contract logs for future blocks, returning a
+// subscription object that can be used to tear down the watcher.
+func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]interface{}) (chan types.Log, event.Subscription, error) {
+	// Don't crash on a lazy user
+	if opts == nil {
+		opts = new(WatchOpts)
+	}
+	// Append the event selector to the query parameters and construct the topic set
+	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+
+	topics, err := makeTopics(query...)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Start the background filtering
+	logs := make(chan types.Log, 128)
+
+	config := ethereum.FilterQuery{
+		Addresses: []common.Address{c.address},
+		Topics:    topics,
+	}
+	if opts.Start != nil {
+		config.FromBlock = new(big.Int).SetUint64(*opts.Start)
+	}
+	sub, err := c.filterer.SubscribeFilterLogs(ensureContext(opts.Context), config, logs)
+	if err != nil {
+		return nil, nil, err
+	}
+	return logs, sub, nil
+}
+
+// UnpackLog unpacks a retrieved log into the provided output structure.
+func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log) error {
+	if len(log.Data) > 0 {
+		if err := c.abi.Unpack(out, event, log.Data); err != nil {
+			return err
+		}
+	}
+	var indexed abi.Arguments
+	for _, arg := range c.abi.Events[event].Inputs {
+		if arg.Indexed {
+			indexed = append(indexed, arg)
+		}
+	}
+	return parseTopics(out, indexed, log.Topics[1:])
+}
+
+// ensureContext is a helper method to ensure a context is not nil, even if the
+// user specified it as such.
 func ensureContext(ctx context.Context) context.Context {
 	if ctx == nil {
 		return context.TODO()

accounts/abi/bind/bind.go

@@ -63,10 +63,11 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			return r
 		}, abis[i])
 
-		// Extract the call and transact methods, and sort them alphabetically
+		// Extract the call and transact methods; events; and sort them alphabetically
 		var (
 			calls     = make(map[string]*tmplMethod)
 			transacts = make(map[string]*tmplMethod)
+			events    = make(map[string]*tmplEvent)
 		)
 		for _, original := range evmABI.Methods {
 			// Normalize the method for capital cases and non-anonymous inputs/outputs
@@ -89,11 +90,33 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			}
 			// Append the methods to the call or transact lists
 			if original.Const {
-				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
+				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
 			} else {
-				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
+				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
 			}
 		}
+		for _, original := range evmABI.Events {
+			// Skip anonymous events as they don't support explicit filtering
+			if original.Anonymous {
+				continue
+			}
+			// Normalize the event for capital cases and non-anonymous outputs
+			normalized := original
+			normalized.Name = methodNormalizer[lang](original.Name)
+
+			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
+			copy(normalized.Inputs, original.Inputs)
+			for j, input := range normalized.Inputs {
+				// Indexed fields are input, non-indexed ones are outputs
+				if input.Indexed {
+					if input.Name == "" {
+						normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
+					}
+				}
+			}
+			// Append the event to the accumulator list
+			events[original.Name] = &tmplEvent{Original: original, Normalized: normalized}
+		}
 		contracts[types[i]] = &tmplContract{
 			Type:        capitalise(types[i]),
 			InputABI:    strings.Replace(strippedABI, "\"", "\\\"", -1),
@@ -101,6 +124,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			Constructor: evmABI.Constructor,
 			Calls:       calls,
 			Transacts:   transacts,
+			Events:      events,
 		}
 	}
 	// Generate the contract template data content and render it
@@ -111,10 +135,11 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	buffer := new(bytes.Buffer)
 
 	funcs := map[string]interface{}{
-		"bindtype":     bindType[lang],
+		"bindtype":      bindType[lang],
-		"namedtype":    namedType[lang],
+		"bindtopictype": bindTopicType[lang],
-		"capitalise":   capitalise,
+		"namedtype":     namedType[lang],
-		"decapitalise": decapitalise,
+		"capitalise":    capitalise,
+		"decapitalise":  decapitalise,
 	}
 	tmpl := template.Must(template.New("").Funcs(funcs).Parse(tmplSource[lang]))
 	if err := tmpl.Execute(buffer, data); err != nil {
@@ -133,7 +158,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 }
 
 // bindType is a set of type binders that convert Solidity types to some supported
-// programming language.
+// programming language types.
 var bindType = map[Lang]func(kind abi.Type) string{
 	LangGo:   bindTypeGo,
 	LangJava: bindTypeJava,
@@ -254,6 +279,33 @@ func bindTypeJava(kind abi.Type) string {
 	}
 }
 
+// bindTopicType is a set of type binders that convert Solidity types to some
+// supported programming language topic types.
+var bindTopicType = map[Lang]func(kind abi.Type) string{
+	LangGo:   bindTopicTypeGo,
+	LangJava: bindTopicTypeJava,
+}
+
+// bindTypeGo converts a Solidity topic type to a Go one. It is almost the same
+// funcionality as for simple types, but dynamic types get converted to hashes.
+func bindTopicTypeGo(kind abi.Type) string {
+	bound := bindTypeGo(kind)
+	if bound == "string" || bound == "[]byte" {
+		bound = "common.Hash"
+	}
+	return bound
+}
+
+// bindTypeGo converts a Solidity topic type to a Java one. It is almost the same
+// funcionality as for simple types, but dynamic types get converted to hashes.
+func bindTopicTypeJava(kind abi.Type) string {
+	bound := bindTypeJava(kind)
+	if bound == "String" || bound == "Bytes" {
+		bound = "Hash"
+	}
+	return bound
+}
+
 // namedType is a set of functions that transform language specific types to
 // named versions that my be used inside method names.
 var namedType = map[Lang]func(string, abi.Type) string{
@@ -321,14 +373,14 @@ func decapitalise(input string) string {
 	return strings.ToLower(input[:1]) + input[1:]
 }
 
-// structured checks whether a method has enough information to return a proper
+// structured checks whether a list of ABI data types has enough information to
-// Go struct or if flat returns are needed.
+// operate through a proper Go struct or if flat returns are needed.
-func structured(method abi.Method) bool {
+func structured(args abi.Arguments) bool {
-	if len(method.Outputs) < 2 {
+	if len(args) < 2 {
 		return false
 	}
 	exists := make(map[string]bool)
-	for _, out := range method.Outputs {
+	for _, out := range args {
 		// If the name is anonymous, we can't organize into a struct
 		if out.Name == "" {
 			return false

accounts/abi/bind/bind_test.go

@@ -148,6 +148,64 @@ var bindTests = []struct {
 			 fmt.Println(str1, str2, res.Str1, res.Str2, err)
 		 }`,
 	},
+	// Tests that named, anonymous and indexed events are handled correctly
+	{
+		`EventChecker`, ``, ``,
+		`
+			[
+				{"type":"event","name":"empty","inputs":[]},
+				{"type":"event","name":"indexed","inputs":[{"name":"addr","type":"address","indexed":true},{"name":"num","type":"int256","indexed":true}]},
+				{"type":"event","name":"mixed","inputs":[{"name":"addr","type":"address","indexed":true},{"name":"num","type":"int256"}]},
+				{"type":"event","name":"anonymous","anonymous":true,"inputs":[]},
+				{"type":"event","name":"dynamic","inputs":[{"name":"idxStr","type":"string","indexed":true},{"name":"idxDat","type":"bytes","indexed":true},{"name":"str","type":"string"},{"name":"dat","type":"bytes"}]}
+			]
+		`,
+		`if e, err := NewEventChecker(common.Address{}, nil); e == nil || err != nil {
+			 t.Fatalf("binding (%v) nil or error (%v) not nil", e, nil)
+		 } else if false { // Don't run, just compile and test types
+			 var (
+				 err  error
+			   res  bool
+				 str  string
+				 dat  []byte
+				 hash common.Hash
+			 )
+			 _, err = e.FilterEmpty(nil)
+			 _, err = e.FilterIndexed(nil, []common.Address{}, []*big.Int{})
+
+			 mit, err := e.FilterMixed(nil, []common.Address{})
+
+			 res = mit.Next()  // Make sure the iterator has a Next method
+			 err = mit.Error() // Make sure the iterator has an Error method
+			 err = mit.Close() // Make sure the iterator has a Close method
+
+			 fmt.Println(mit.Event.Raw.BlockHash) // Make sure the raw log is contained within the results
+			 fmt.Println(mit.Event.Num)           // Make sure the unpacked non-indexed fields are present
+			 fmt.Println(mit.Event.Addr)          // Make sure the reconstructed indexed fields are present
+
+			 dit, err := e.FilterDynamic(nil, []string{}, [][]byte{})
+
+			 str  = dit.Event.Str    // Make sure non-indexed strings retain their type
+			 dat  = dit.Event.Dat    // Make sure non-indexed bytes retain their type
+			 hash = dit.Event.IdxStr // Make sure indexed strings turn into hashes
+			 hash = dit.Event.IdxDat // Make sure indexed bytes turn into hashes
+
+			 sink := make(chan *EventCheckerMixed)
+			 sub, err := e.WatchMixed(nil, sink, []common.Address{})
+			 defer sub.Unsubscribe()
+
+			 event := <-sink
+			 fmt.Println(event.Raw.BlockHash) // Make sure the raw log is contained within the results
+			 fmt.Println(event.Num)           // Make sure the unpacked non-indexed fields are present
+			 fmt.Println(event.Addr)          // Make sure the reconstructed indexed fields are present
+
+			 fmt.Println(res, str, dat, hash, err)
+		 }
+		 // Run a tiny reflection test to ensure disallowed methods don't appear
+		 if _, ok := reflect.TypeOf(&EventChecker{}).MethodByName("FilterAnonymous"); ok {
+		 	t.Errorf("binding has disallowed method (FilterAnonymous)")
+		 }`,
+	},
 	// Test that contract interactions (deploy, transact and call) generate working code
 	{
 		`Interactor`,
@@ -508,6 +566,177 @@ var bindTests = []struct {
 			fmt.Println(a, b, err)
 		`,
 	},
+	// Tests that logs can be successfully filtered and decoded.
+	{
+		`Eventer`,
+		`
+			contract Eventer {
+					event SimpleEvent (
+					address indexed Addr,
+					bytes32 indexed Id,
+					bool    indexed Flag,
+					uint    Value
+				);
+				function raiseSimpleEvent(address addr, bytes32 id, bool flag, uint value) {
+					SimpleEvent(addr, id, flag, value);
+				}
+
+				event NodataEvent (
+					uint   indexed Number,
+					int16  indexed Short,
+					uint32 indexed Long
+				);
+				function raiseNodataEvent(uint number, int16 short, uint32 long) {
+					NodataEvent(number, short, long);
+				}
+
+				event DynamicEvent (
+					string indexed IndexedString,
+					bytes  indexed IndexedBytes,
+					string NonIndexedString,
+					bytes  NonIndexedBytes
+				);
+				function raiseDynamicEvent(string str, bytes blob) {
+					DynamicEvent(str, blob, str, blob);
+				}
+			}
+		`,
+		`6060604052341561000f57600080fd5b61042c8061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063528300ff1461005c578063630c31e2146100fc578063c7d116dd14610156575b600080fd5b341561006757600080fd5b6100fa600480803590602001908201803590602001908080601f0160208091040260200160405190810160405280939291908181526020018383808284378201915050505050509190803590602001908201803590602001908080601f01602080910402602001604051908101604052809392919081815260200183838082843782019150505050505091905050610194565b005b341561010757600080fd5b610154600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091908035600019169060200190919080351515906020019091908035906020019091905050610367565b005b341561016157600080fd5b610192600480803590602001909190803560010b90602001909190803563ffffffff169060200190919050506103c3565b005b806040518082805190602001908083835b6020831015156101ca57805182526020820191506020810190506020830392506101a5565b6001836020036101000a0380198251168184511680821785525050505050509050019150506040518091039020826040518082805190602001908083835b60208310151561022d5780518252602082019150602081019050602083039250610208565b6001836020036101000a03801982511681845116808217855250505050505090500191505060405180910390207f3281fd4f5e152dd3385df49104a3f633706e21c9e80672e88d3bcddf33101f008484604051808060200180602001838103835285818151815260200191508051906020019080838360005b838110156102c15780820151818401526020810190506102a6565b50505050905090810190601f1680156102ee5780820380516001836020036101000a031916815260200191505b50838103825284818151815260200191508051906020019080838360005b8381101561032757808201518184015260208101905061030c565b50505050905090810190601f1680156103545780820380516001836020036101000a031916815260200191505b5094505050505060405180910390a35050565b81151583600019168573ffffffffffffffffffffffffffffffffffffffff167f1f097de4289df643bd9c11011cc61367aa12983405c021056e706eb5ba1250c8846040518082815260200191505060405180910390a450505050565b8063ffffffff168260010b847f3ca7f3a77e5e6e15e781850bc82e32adfa378a2a609370db24b4d0fae10da2c960405160405180910390a45050505600a165627a7a72305820d1f8a8bbddbc5bb29f285891d6ae1eef8420c52afdc05e1573f6114d8e1714710029`,
+		`[{"constant":false,"inputs":[{"name":"str","type":"string"},{"name":"blob","type":"bytes"}],"name":"raiseDynamicEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"},{"name":"id","type":"bytes32"},{"name":"flag","type":"bool"},{"name":"value","type":"uint256"}],"name":"raiseSimpleEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"number","type":"uint256"},{"name":"short","type":"int16"},{"name":"long","type":"uint32"}],"name":"raiseNodataEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Addr","type":"address"},{"indexed":true,"name":"Id","type":"bytes32"},{"indexed":true,"name":"Flag","type":"bool"},{"indexed":false,"name":"Value","type":"uint256"}],"name":"SimpleEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Number","type":"uint256"},{"indexed":true,"name":"Short","type":"int16"},{"indexed":true,"name":"Long","type":"uint32"}],"name":"NodataEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedString","type":"string"},{"indexed":true,"name":"IndexedBytes","type":"bytes"},{"indexed":false,"name":"NonIndexedString","type":"string"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes"}],"name":"DynamicEvent","type":"event"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy an eventer contract
+			_, _, eventer, err := DeployEventer(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy eventer contract: %v", err)
+			}
+			sim.Commit()
+
+			// Inject a few events into the contract, gradually more in each block
+			for i := 1; i <= 3; i++ {
+				for j := 1; j <= i; j++ {
+					if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil {
+						t.Fatalf("block %d, event %d: raise failed: %v", i, j, err)
+					}
+				}
+				sim.Commit()
+			}
+			// Test filtering for certain events and ensure they can be found
+			sit, err := eventer.FilterSimpleEvent(nil, []common.Address{common.Address{1}, common.Address{3}}, [][32]byte{{byte(1)}, {byte(2)}, {byte(3)}}, []bool{true})
+			if err != nil {
+				t.Fatalf("failed to filter for simple events: %v", err)
+			}
+			defer sit.Close()
+
+			sit.Next()
+			if sit.Event.Value.Uint64() != 11 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {11, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 21 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {21, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 31 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {31, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 33 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {33, true}", sit.Event)
+			}
+
+			if sit.Next() {
+				t.Errorf("unexpected simple event found: %+v", sit.Event)
+			}
+			if err = sit.Error(); err != nil {
+				t.Fatalf("simple event iteration failed: %v", err)
+			}
+			// Test raising and filtering for an event with no data component
+			if _, err := eventer.RaiseNodataEvent(auth, big.NewInt(314), 141, 271); err != nil {
+				t.Fatalf("failed to raise nodata event: %v", err)
+			}
+			sim.Commit()
+
+			nit, err := eventer.FilterNodataEvent(nil, []*big.Int{big.NewInt(314)}, []int16{140, 141, 142}, []uint32{271})
+			if err != nil {
+				t.Fatalf("failed to filter for nodata events: %v", err)
+			}
+			defer nit.Close()
+
+			if !nit.Next() {
+				t.Fatalf("nodata log not found: %v", nit.Error())
+			}
+			if nit.Event.Number.Uint64() != 314 {
+				t.Errorf("nodata log content mismatch: have %v, want 314", nit.Event.Number)
+			}
+			if nit.Next() {
+				t.Errorf("unexpected nodata event found: %+v", nit.Event)
+			}
+			if err = nit.Error(); err != nil {
+				t.Fatalf("nodata event iteration failed: %v", err)
+			}
+			// Test raising and filtering for events with dynamic indexed components
+			if _, err := eventer.RaiseDynamicEvent(auth, "Hello", []byte("World")); err != nil {
+				t.Fatalf("failed to raise dynamic event: %v", err)
+			}
+			sim.Commit()
+
+			dit, err := eventer.FilterDynamicEvent(nil, []string{"Hi", "Hello", "Bye"}, [][]byte{[]byte("World")})
+			if err != nil {
+				t.Fatalf("failed to filter for dynamic events: %v", err)
+			}
+			defer dit.Close()
+
+			if !dit.Next() {
+				t.Fatalf("dynamic log not found: %v", dit.Error())
+			}
+			if dit.Event.NonIndexedString != "Hello" || string(dit.Event.NonIndexedBytes) != "World" ||	dit.Event.IndexedString != common.HexToHash("0x06b3dfaec148fb1bb2b066f10ec285e7c9bf402ab32aa78a5d38e34566810cd2") || dit.Event.IndexedBytes != common.HexToHash("0xf2208c967df089f60420785795c0a9ba8896b0f6f1867fa7f1f12ad6f79c1a18") {
+				t.Errorf("dynamic log content mismatch: have %v, want {'0x06b3dfaec148fb1bb2b066f10ec285e7c9bf402ab32aa78a5d38e34566810cd2, '0xf2208c967df089f60420785795c0a9ba8896b0f6f1867fa7f1f12ad6f79c1a18', 'Hello', 'World'}", dit.Event)
+			}
+			if dit.Next() {
+				t.Errorf("unexpected dynamic event found: %+v", dit.Event)
+			}
+			if err = dit.Error(); err != nil {
+				t.Fatalf("dynamic event iteration failed: %v", err)
+			}
+			// Test subscribing to an event and raising it afterwards
+			ch := make(chan *EventerSimpleEvent, 16)
+			sub, err := eventer.WatchSimpleEvent(nil, ch, nil, nil, nil)
+			if err != nil {
+				t.Fatalf("failed to subscribe to simple events: %v", err)
+			}
+			if _, err := eventer.RaiseSimpleEvent(auth, common.Address{255}, [32]byte{255}, true, big.NewInt(255)); err != nil {
+				t.Fatalf("failed to raise subscribed simple event: %v", err)
+			}
+			sim.Commit()
+
+			select {
+			case event := <-ch:
+				if event.Value.Uint64() != 255 {
+					t.Errorf("simple log content mismatch: have %v, want 255", event)
+				}
+			case <-time.After(250 * time.Millisecond):
+				t.Fatalf("subscribed simple event didn't arrive")
+			}
+			// Unsubscribe from the event and make sure we're not delivered more
+			sub.Unsubscribe()
+
+			if _, err := eventer.RaiseSimpleEvent(auth, common.Address{254}, [32]byte{254}, true, big.NewInt(254)); err != nil {
+				t.Fatalf("failed to raise subscribed simple event: %v", err)
+			}
+			sim.Commit()
+
+			select {
+			case event := <-ch:
+				t.Fatalf("unsubscribed simple event arrived: %v", event)
+			case <-time.After(250 * time.Millisecond):
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and
@@ -559,7 +788,7 @@ func TestBindings(t *testing.T) {
 		}
 	}
 	// Test the entire package and report any failures
-	cmd := exec.Command(gocmd, "test", "-v")
+	cmd := exec.Command(gocmd, "test", "-v", "-count", "1")
 	cmd.Dir = pkg
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatalf("failed to run binding test: %v\n%s", err, out)

accounts/abi/bind/template.go

@@ -32,6 +32,7 @@ type tmplContract struct {
 	Constructor abi.Method             // Contract constructor for deploy parametrization
 	Calls       map[string]*tmplMethod // Contract calls that only read state data
 	Transacts   map[string]*tmplMethod // Contract calls that write state data
+	Events      map[string]*tmplEvent  // Contract events accessors
 }
 
 // tmplMethod is a wrapper around an abi.Method that contains a few preprocessed
@@ -39,7 +40,13 @@ type tmplContract struct {
 type tmplMethod struct {
 	Original   abi.Method // Original method as parsed by the abi package
 	Normalized abi.Method // Normalized version of the parsed method (capitalized names, non-anonymous args/returns)
-	Structured bool       // Whether the returns should be accumulated into a contract
+	Structured bool       // Whether the returns should be accumulated into a struct
+}
+
+// tmplEvent is a wrapper around an a
+type tmplEvent struct {
+	Original   abi.Event // Original event as parsed by the abi package
+	Normalized abi.Event // Normalized version of the parsed fields
 }
 
 // tmplSource is language to template mapping containing all the supported
@@ -75,7 +82,7 @@ package {{.Package}}
 		  if err != nil {
 		    return common.Address{}, nil, nil, err
 		  }
-		  return address, tx, &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract} }, nil
+		  return address, tx, &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract}, {{.Type}}Filterer: {{.Type}}Filterer{contract: contract} }, nil
 		}
 	{{end}}
 
@@ -83,6 +90,7 @@ package {{.Package}}
 	type {{.Type}} struct {
 	  {{.Type}}Caller     // Read-only binding to the contract
 	  {{.Type}}Transactor // Write-only binding to the contract
+		{{.Type}}Filterer   // Log filterer for contract events
 	}
 
 	// {{.Type}}Caller is an auto generated read-only Go binding around an Ethereum contract.
@@ -95,6 +103,11 @@ package {{.Package}}
 	  contract *bind.BoundContract // Generic contract wrapper for the low level calls
 	}
 
+	// {{.Type}}Filterer is an auto generated log filtering Go binding around an Ethereum contract events.
+	type {{.Type}}Filterer struct {
+	  contract *bind.BoundContract // Generic contract wrapper for the low level calls
+	}
+
 	// {{.Type}}Session is an auto generated Go binding around an Ethereum contract,
 	// with pre-set call and transact options.
 	type {{.Type}}Session struct {
@@ -134,16 +147,16 @@ package {{.Package}}
 
 	// New{{.Type}} creates a new instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}(address common.Address, backend bind.ContractBackend) (*{{.Type}}, error) {
-	  contract, err := bind{{.Type}}(address, backend, backend)
+	  contract, err := bind{{.Type}}(address, backend, backend, backend)
 	  if err != nil {
 	    return nil, err
 	  }
-	  return &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract} }, nil
+	  return &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract}, {{.Type}}Filterer: {{.Type}}Filterer{contract: contract} }, nil
 	}
 
 	// New{{.Type}}Caller creates a new read-only instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}Caller(address common.Address, caller bind.ContractCaller) (*{{.Type}}Caller, error) {
-	  contract, err := bind{{.Type}}(address, caller, nil)
+	  contract, err := bind{{.Type}}(address, caller, nil, nil)
 	  if err != nil {
 	    return nil, err
 	  }
@@ -152,20 +165,29 @@ package {{.Package}}
 
 	// New{{.Type}}Transactor creates a new write-only instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}Transactor(address common.Address, transactor bind.ContractTransactor) (*{{.Type}}Transactor, error) {
-	  contract, err := bind{{.Type}}(address, nil, transactor)
+	  contract, err := bind{{.Type}}(address, nil, transactor, nil)
 	  if err != nil {
 	    return nil, err
 	  }
 	  return &{{.Type}}Transactor{contract: contract}, nil
 	}
 
+	// New{{.Type}}Filterer creates a new log filterer instance of {{.Type}}, bound to a specific deployed contract.
+ 	func New{{.Type}}Filterer(address common.Address, filterer bind.ContractFilterer) (*{{.Type}}Filterer, error) {
+ 	  contract, err := bind{{.Type}}(address, nil, nil, filterer)
+ 	  if err != nil {
+ 	    return nil, err
+ 	  }
+ 	  return &{{.Type}}Filterer{contract: contract}, nil
+ 	}
+
 	// bind{{.Type}} binds a generic wrapper to an already deployed contract.
-	func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) {
+	func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
 	  parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI))
 	  if err != nil {
 	    return nil, err
 	  }
-	  return bind.NewBoundContract(address, parsed, caller, transactor), nil
+	  return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
 	}
 
 	// Call invokes the (constant) contract method with params as input values and
@@ -263,6 +285,137 @@ package {{.Package}}
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 	{{end}}
+
+	{{range .Events}}
+		// {{$contract.Type}}{{.Normalized.Name}}Iterator is returned from Filter{{.Normalized.Name}} and is used to iterate over the raw logs and unpacked data for {{.Normalized.Name}} events raised by the {{$contract.Type}} contract.
+		type {{$contract.Type}}{{.Normalized.Name}}Iterator struct {
+			Event *{{$contract.Type}}{{.Normalized.Name}} // Event containing the contract specifics and raw log
+
+			contract *bind.BoundContract // Generic contract to use for unpacking event data
+			event    string              // Event name to use for unpacking event data
+
+			logs chan types.Log        // Log channel receiving the found contract events
+			sub  ethereum.Subscription // Subscription for errors, completion and termination
+			done bool                  // Whether the subscription completed delivering logs
+			fail error                 // Occurred error to stop iteration
+		}
+		// Next advances the iterator to the subsequent event, returning whether there
+		// are any more events found. In case of a retrieval or parsing error, false is
+		// returned and Error() can be queried for the exact failure.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Next() bool {
+			// If the iterator failed, stop iterating
+			if (it.fail != nil) {
+				return false
+			}
+			// If the iterator completed, deliver directly whatever's available
+			if (it.done) {
+				select {
+				case log := <-it.logs:
+					it.Event = new({{$contract.Type}}{{.Normalized.Name}})
+					if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+						it.fail = err
+						return false
+					}
+					it.Event.Raw = log
+					return true
+
+				default:
+					return false
+				}
+			}
+			// Iterator still in progress, wait for either a data or an error event
+			select {
+			case log := <-it.logs:
+				it.Event = new({{$contract.Type}}{{.Normalized.Name}})
+				if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+					it.fail = err
+					return false
+				}
+				it.Event.Raw = log
+				return true
+
+			case err := <-it.sub.Err():
+				it.done = true
+				it.fail = err
+				return it.Next()
+			}
+		}
+		// Error returns any retrieval or parsing error occurred during filtering.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Error() error {
+			return it.fail
+		}
+		// Close terminates the iteration process, releasing any pending underlying
+		// resources.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Close() error {
+			it.sub.Unsubscribe()
+			return nil
+		}
+
+		// {{$contract.Type}}{{.Normalized.Name}} represents a {{.Normalized.Name}} event raised by the {{$contract.Type}} contract.
+		type {{$contract.Type}}{{.Normalized.Name}} struct { {{range .Normalized.Inputs}}
+			{{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type}}{{else}}{{bindtype .Type}}{{end}}; {{end}}
+			Raw types.Log // Blockchain specific contextual infos
+		}
+
+		// Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		//
+		// Solidity: {{.Original.String}}
+ 		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) {
+			{{range .Normalized.Inputs}}
+			{{if .Indexed}}var {{.Name}}Rule []interface{}
+			for _, {{.Name}}Item := range {{.Name}} {
+				{{.Name}}Rule = append({{.Name}}Rule, {{.Name}}Item)
+			}{{end}}{{end}}
+
+			logs, sub, err := _{{$contract.Type}}.contract.FilterLogs(opts, "{{.Original.Name}}"{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}}Rule{{end}}{{end}})
+			if err != nil {
+				return nil, err
+			}
+			return &{{$contract.Type}}{{.Normalized.Name}}Iterator{contract: _{{$contract.Type}}.contract, event: "{{.Original.Name}}", logs: logs, sub: sub}, nil
+ 		}
+
+		// Watch{{.Normalized.Name}} is a free log subscription operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		//
+		// Solidity: {{.Original.String}}
+		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (event.Subscription, error) {
+			{{range .Normalized.Inputs}}
+			{{if .Indexed}}var {{.Name}}Rule []interface{}
+			for _, {{.Name}}Item := range {{.Name}} {
+				{{.Name}}Rule = append({{.Name}}Rule, {{.Name}}Item)
+			}{{end}}{{end}}
+
+			logs, sub, err := _{{$contract.Type}}.contract.WatchLogs(opts, "{{.Original.Name}}"{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}}Rule{{end}}{{end}})
+			if err != nil {
+				return nil, err
+			}
+			return event.NewSubscription(func(quit <-chan struct{}) error {
+				defer sub.Unsubscribe()
+				for {
+					select {
+					case log := <-logs:
+						// New log arrived, parse the event and forward to the user
+						event := new({{$contract.Type}}{{.Normalized.Name}})
+						if err := _{{$contract.Type}}.contract.UnpackLog(event, "{{.Original.Name}}", log); err != nil {
+							return err
+						}
+						event.Raw = log
+
+						select {
+						case sink <- event:
+						case err := <-sub.Err():
+							return err
+						case <-quit:
+							return nil
+						}
+					case err := <-sub.Err():
+						return err
+					case <-quit:
+						return nil
+					}
+				}
+			}), nil
+		}
+ 	{{end}}
 {{end}}
 `
 

accounts/abi/bind/topics.go

@@ -0,0 +1,189 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bind
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// makeTopics converts a filter query argument list into a filter topic set.
+func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
+	topics := make([][]common.Hash, len(query))
+	for i, filter := range query {
+		for _, rule := range filter {
+			var topic common.Hash
+
+			// Try to generate the topic based on simple types
+			switch rule := rule.(type) {
+			case common.Hash:
+				copy(topic[:], rule[:])
+			case common.Address:
+				copy(topic[common.HashLength-common.AddressLength:], rule[:])
+			case *big.Int:
+				blob := rule.Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case bool:
+				if rule {
+					topic[common.HashLength-1] = 1
+				}
+			case int8:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int16:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int32:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int64:
+				blob := big.NewInt(rule).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint8:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint16:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint32:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint64:
+				blob := new(big.Int).SetUint64(rule).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case string:
+				hash := crypto.Keccak256Hash([]byte(rule))
+				copy(topic[:], hash[:])
+			case []byte:
+				hash := crypto.Keccak256Hash(rule)
+				copy(topic[:], hash[:])
+
+			default:
+				// Attempt to generate the topic from funky types
+				val := reflect.ValueOf(rule)
+
+				switch {
+				case val.Kind() == reflect.Array && reflect.TypeOf(rule).Elem().Kind() == reflect.Uint8:
+					reflect.Copy(reflect.ValueOf(topic[common.HashLength-val.Len():]), val)
+
+				default:
+					return nil, fmt.Errorf("unsupported indexed type: %T", rule)
+				}
+			}
+			topics[i] = append(topics[i], topic)
+		}
+	}
+	return topics, nil
+}
+
+// Big batch of reflect types for topic reconstruction.
+var (
+	reflectHash    = reflect.TypeOf(common.Hash{})
+	reflectAddress = reflect.TypeOf(common.Address{})
+	reflectBigInt  = reflect.TypeOf(new(big.Int))
+)
+
+// parseTopics converts the indexed topic fields into actual log field values.
+//
+// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
+// hashes as the topic value!
+func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) error {
+	// Sanity check that the fields and topics match up
+	if len(fields) != len(topics) {
+		return errors.New("topic/field count mismatch")
+	}
+	// Iterate over all the fields and reconstruct them from topics
+	for _, arg := range fields {
+		if !arg.Indexed {
+			return errors.New("non-indexed field in topic reconstruction")
+		}
+		field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
+
+		// Try to parse the topic back into the fields based on primitive types
+		switch field.Kind() {
+		case reflect.Bool:
+			if topics[0][common.HashLength-1] == 1 {
+				field.Set(reflect.ValueOf(true))
+			}
+		case reflect.Int8:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int8(num.Int64())))
+
+		case reflect.Int16:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int16(num.Int64())))
+
+		case reflect.Int32:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int32(num.Int64())))
+
+		case reflect.Int64:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(num.Int64()))
+
+		case reflect.Uint8:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint8(num.Uint64())))
+
+		case reflect.Uint16:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint16(num.Uint64())))
+
+		case reflect.Uint32:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint32(num.Uint64())))
+
+		case reflect.Uint64:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(num.Uint64()))
+
+		default:
+			// Ran out of plain primitive types, try custom types
+			switch field.Type() {
+			case reflectHash: // Also covers all dynamic types
+				field.Set(reflect.ValueOf(topics[0]))
+
+			case reflectAddress:
+				var addr common.Address
+				copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
+				field.Set(reflect.ValueOf(addr))
+
+			case reflectBigInt:
+				num := new(big.Int).SetBytes(topics[0][:])
+				field.Set(reflect.ValueOf(num))
+
+			default:
+				// Ran out of custom types, try the crazies
+				switch {
+				case arg.Type.T == abi.FixedBytesTy:
+					reflect.Copy(field, reflect.ValueOf(topics[0][common.HashLength-arg.Type.Size:]))
+
+				default:
+					return fmt.Errorf("unsupported indexed type: %v", arg.Type)
+				}
+			}
+		}
+		topics = topics[1:]
+	}
+	return nil
+}

accounts/abi/event.go

@@ -33,6 +33,17 @@ type Event struct {
 	Inputs    Arguments
 }
 
+func (event Event) String() string {
+	inputs := make([]string, len(event.Inputs))
+	for i, input := range event.Inputs {
+		inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
+		if input.Indexed {
+			inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type)
+		}
+	}
+	return fmt.Sprintf("event %v(%v)", event.Name, strings.Join(inputs, ", "))
+}
+
 // Id returns the canonical representation of the event's signature used by the
 // abi definition to identify event names and types.
 func (e Event) Id() common.Hash {

accounts/errors.go

@@ -62,7 +62,7 @@ func NewAuthNeededError(needed string) error {
 	}
 }
 
-// Error implements the standard error interfacel.
+// Error implements the standard error interface.
 func (err *AuthNeededError) Error() string {
 	return fmt.Sprintf("authentication needed: %s", err.Needed)
 }

cmd/bootnode/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/ecdsa"
 	"flag"
 	"fmt"
+	"net"
 	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -96,12 +97,32 @@ func main() {
 		}
 	}
 
+	addr, err := net.ResolveUDPAddr("udp", *listenAddr)
+	if err != nil {
+		utils.Fatalf("-ResolveUDPAddr: %v", err)
+	}
+	conn, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		utils.Fatalf("-ListenUDP: %v", err)
+	}
+
+	realaddr := conn.LocalAddr().(*net.UDPAddr)
+	if natm != nil {
+		if !realaddr.IP.IsLoopback() {
+			go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
+		}
+		// TODO: react to external IP changes over time.
+		if ext, err := natm.ExternalIP(); err == nil {
+			realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
+		}
+	}
+
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
-		if _, err := discover.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil {
+		if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/ethkey/generate.go

@@ -1,8 +1,23 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
 package main
 
 import (
 	"crypto/ecdsa"
-	"crypto/rand"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -26,16 +41,16 @@ var commandGenerate = cli.Command{
 	ArgsUsage: "[ <keyfile> ]",
 	Description: `
 Generate a new keyfile.
-If you want to use an existing private key to use in the keyfile, it can be 
+
-specified by setting --privatekey with the location of the file containing the 
+If you want to encrypt an existing private key, it can be specified by setting
-private key.`,
+--privatekey with the location of the file containing the private key.
+`,
 	Flags: []cli.Flag{
 		passphraseFlag,
 		jsonFlag,
 		cli.StringFlag{
-			Name: "privatekey",
+			Name:  "privatekey",
-			Usage: "the file from where to read the private key to " +
+			Usage: "file containing a raw private key to encrypt",
-				"generate a keyfile for",
 		},
 	},
 	Action: func(ctx *cli.Context) error {
@@ -51,32 +66,19 @@ private key.`,
 		}
 
 		var privateKey *ecdsa.PrivateKey
-
+		var err error
-		// First check if a private key file is provided.
+		if file := ctx.String("privatekey"); file != "" {
-		privateKeyFile := ctx.String("privatekey")
+			// Load private key from file.
-		if privateKeyFile != "" {
+			privateKey, err = crypto.LoadECDSA(file)
-			privateKeyBytes, err := ioutil.ReadFile(privateKeyFile)
 			if err != nil {
-				utils.Fatalf("Failed to read the private key file '%s': %v",
+				utils.Fatalf("Can't load private key: %v", err)
-					privateKeyFile, err)
 			}
-
+		} else {
-			pk, err := crypto.HexToECDSA(string(privateKeyBytes))
+			// If not loaded, generate random.
-			if err != nil {
+			privateKey, err = crypto.GenerateKey()
-				utils.Fatalf(
-					"Could not construct ECDSA private key from file content: %v",
-					err)
-			}
-			privateKey = pk
-		}
-
-		// If not loaded, generate random.
-		if privateKey == nil {
-			pk, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader)
 			if err != nil {
 				utils.Fatalf("Failed to generate random private key: %v", err)
 			}
-			privateKey = pk
 		}
 
 		// Create the keyfile object with a random UUID.
@@ -89,8 +91,7 @@ private key.`,
 
 		// Encrypt key with passphrase.
 		passphrase := getPassPhrase(ctx, true)
-		keyjson, err := keystore.EncryptKey(key, passphrase,
+		keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP)
-			keystore.StandardScryptN, keystore.StandardScryptP)
 		if err != nil {
 			utils.Fatalf("Error encrypting key: %v", err)
 		}
@@ -110,7 +111,7 @@ private key.`,
 		if ctx.Bool(jsonFlag.Name) {
 			mustPrintJSON(out)
 		} else {
-			fmt.Println("Address:       ", out.Address)
+			fmt.Println("Address:", out.Address)
 		}
 		return nil
 	},

cmd/ethkey/inspect.go

@@ -23,6 +23,7 @@ var commandInspect = cli.Command{
 	ArgsUsage: "<keyfile>",
 	Description: `
 Print various information about the keyfile.
+
 Private key information can be printed by using the --private flag;
 make sure to use this feature with great caution!`,
 	Flags: []cli.Flag{

cmd/ethkey/main.go

@@ -28,30 +28,11 @@ const (
 	defaultKeyfileName = "keyfile.json"
 )
 
-var (
+// Git SHA1 commit hash of the release (set via linker flags)
-	gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags)
+var gitCommit = ""
 
-	app *cli.App // the main app instance
+var app *cli.App
-)
 
-var ( // Commonly used command line flags.
-	passphraseFlag = cli.StringFlag{
-		Name:  "passwordfile",
-		Usage: "the file that contains the passphrase for the keyfile",
-	}
-
-	jsonFlag = cli.BoolFlag{
-		Name:  "json",
-		Usage: "output JSON instead of human-readable format",
-	}
-
-	messageFlag = cli.StringFlag{
-		Name:  "message",
-		Usage: "the file that contains the message to sign/verify",
-	}
-)
-
-// Configure the app instance.
 func init() {
 	app = utils.NewApp(gitCommit, "an Ethereum key manager")
 	app.Commands = []cli.Command{
@@ -62,6 +43,22 @@ func init() {
 	}
 }
 
+// Commonly used command line flags.
+var (
+	passphraseFlag = cli.StringFlag{
+		Name:  "passwordfile",
+		Usage: "the file that contains the passphrase for the keyfile",
+	}
+	jsonFlag = cli.BoolFlag{
+		Name:  "json",
+		Usage: "output JSON instead of human-readable format",
+	}
+	messageFlag = cli.StringFlag{
+		Name:  "message",
+		Usage: "the file that contains the message to sign/verify",
+	}
+)
+
 func main() {
 	if err := app.Run(os.Args); err != nil {
 		fmt.Fprintln(os.Stderr, err)

cmd/ethkey/message.go

@@ -1,11 +1,25 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
 package main
 
 import (
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
-	"os"
-	"strings"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -18,26 +32,33 @@ type outputSign struct {
 	Signature string
 }
 
+var msgfileFlag = cli.StringFlag{
+	Name:  "msgfile",
+	Usage: "file containing the message to sign/verify",
+}
+
 var commandSignMessage = cli.Command{
 	Name:      "signmessage",
 	Usage:     "sign a message",
-	ArgsUsage: "<keyfile> <message/file>",
+	ArgsUsage: "<keyfile> <message>",
 	Description: `
 Sign the message with a keyfile.
-It is possible to refer to a file containing the message.`,
+
+To sign a message contained in a file, use the --msgfile flag.
+`,
 	Flags: []cli.Flag{
 		passphraseFlag,
 		jsonFlag,
+		msgfileFlag,
 	},
 	Action: func(ctx *cli.Context) error {
-		keyfilepath := ctx.Args().First()
+		message := getMessage(ctx, 1)
-		message := []byte(ctx.Args().Get(1))
 
 		// Load the keyfile.
+		keyfilepath := ctx.Args().First()
 		keyjson, err := ioutil.ReadFile(keyfilepath)
 		if err != nil {
-			utils.Fatalf("Failed to read the keyfile at '%s': %v",
+			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
-				keyfilepath, err)
 		}
 
 		// Decrypt key with passphrase.
@@ -47,29 +68,15 @@ It is possible to refer to a file containing the message.`,
 			utils.Fatalf("Error decrypting key: %v", err)
 		}
 
-		if len(message) == 0 {
-			utils.Fatalf("A message must be provided")
-		}
-		// Read message if file.
-		if _, err := os.Stat(string(message)); err == nil {
-			message, err = ioutil.ReadFile(string(message))
-			if err != nil {
-				utils.Fatalf("Failed to read the message file: %v", err)
-			}
-		}
-
 		signature, err := crypto.Sign(signHash(message), key.PrivateKey)
 		if err != nil {
 			utils.Fatalf("Failed to sign message: %v", err)
 		}
-
+		out := outputSign{Signature: hex.EncodeToString(signature)}
-		out := outputSign{
-			Signature: hex.EncodeToString(signature),
-		}
 		if ctx.Bool(jsonFlag.Name) {
 			mustPrintJSON(out)
 		} else {
-			fmt.Println("Signature: ", out.Signature)
+			fmt.Println("Signature:", out.Signature)
 		}
 		return nil
 	},
@@ -84,53 +91,40 @@ type outputVerify struct {
 var commandVerifyMessage = cli.Command{
 	Name:      "verifymessage",
 	Usage:     "verify the signature of a signed message",
-	ArgsUsage: "<address> <signature> <message/file>",
+	ArgsUsage: "<address> <signature> <message>",
 	Description: `
 Verify the signature of the message.
 It is possible to refer to a file containing the message.`,
 	Flags: []cli.Flag{
 		jsonFlag,
+		msgfileFlag,
 	},
 	Action: func(ctx *cli.Context) error {
 		addressStr := ctx.Args().First()
 		signatureHex := ctx.Args().Get(1)
-		message := []byte(ctx.Args().Get(2))
+		message := getMessage(ctx, 2)
 
-		// Determine whether it is a keyfile, public key or address.
 		if !common.IsHexAddress(addressStr) {
 			utils.Fatalf("Invalid address: %s", addressStr)
 		}
 		address := common.HexToAddress(addressStr)
-
 		signature, err := hex.DecodeString(signatureHex)
 		if err != nil {
 			utils.Fatalf("Signature encoding is not hexadecimal: %v", err)
 		}
 
-		if len(message) == 0 {
-			utils.Fatalf("A message must be provided")
-		}
-		// Read message if file.
-		if _, err := os.Stat(string(message)); err == nil {
-			message, err = ioutil.ReadFile(string(message))
-			if err != nil {
-				utils.Fatalf("Failed to read the message file: %v", err)
-			}
-		}
-
 		recoveredPubkey, err := crypto.SigToPub(signHash(message), signature)
 		if err != nil || recoveredPubkey == nil {
 			utils.Fatalf("Signature verification failed: %v", err)
 		}
 		recoveredPubkeyBytes := crypto.FromECDSAPub(recoveredPubkey)
 		recoveredAddress := crypto.PubkeyToAddress(*recoveredPubkey)
-
 		success := address == recoveredAddress
 
 		out := outputVerify{
 			Success:            success,
 			RecoveredPublicKey: hex.EncodeToString(recoveredPubkeyBytes),
-			RecoveredAddress:   strings.ToLower(recoveredAddress.Hex()),
+			RecoveredAddress:   recoveredAddress.Hex(),
 		}
 		if ctx.Bool(jsonFlag.Name) {
 			mustPrintJSON(out)
@@ -140,9 +134,26 @@ It is possible to refer to a file containing the message.`,
 			} else {
 				fmt.Println("Signature verification failed!")
 			}
-			fmt.Println("Recovered public key: ", out.RecoveredPublicKey)
+			fmt.Println("Recovered public key:", out.RecoveredPublicKey)
-			fmt.Println("Recovered address: ", out.RecoveredAddress)
+			fmt.Println("Recovered address:", out.RecoveredAddress)
 		}
 		return nil
 	},
 }
+
+func getMessage(ctx *cli.Context, msgarg int) []byte {
+	if file := ctx.String("msgfile"); file != "" {
+		if len(ctx.Args()) > msgarg {
+			utils.Fatalf("Can't use --msgfile and message argument at the same time.")
+		}
+		msg, err := ioutil.ReadFile(file)
+		if err != nil {
+			utils.Fatalf("Can't read message file: %v", err)
+		}
+		return msg
+	} else if len(ctx.Args()) == msgarg+1 {
+		return []byte(ctx.Args().Get(msgarg))
+	}
+	utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, len(ctx.Args()))
+	return nil
+}

cmd/ethkey/message_test.go

@@ -0,0 +1,70 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestMessageSignVerify(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "ethkey-test")
+	if err != nil {
+		t.Fatal("Can't create temporary directory:", err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	keyfile := filepath.Join(tmpdir, "the-keyfile")
+	message := "test message"
+
+	// Create the key.
+	generate := runEthkey(t, "generate", keyfile)
+	generate.Expect(`
+!! Unsupported terminal, password will be echoed.
+Passphrase: {{.InputLine "foobar"}}
+Repeat passphrase: {{.InputLine "foobar"}}
+`)
+	_, matches := generate.ExpectRegexp(`Address: (0x[0-9a-fA-F]{40})\n`)
+	address := matches[1]
+	generate.ExpectExit()
+
+	// Sign a message.
+	sign := runEthkey(t, "signmessage", keyfile, message)
+	sign.Expect(`
+!! Unsupported terminal, password will be echoed.
+Passphrase: {{.InputLine "foobar"}}
+`)
+	_, matches = sign.ExpectRegexp(`Signature: ([0-9a-f]+)\n`)
+	signature := matches[1]
+	sign.ExpectExit()
+
+	// Verify the message.
+	verify := runEthkey(t, "verifymessage", address, signature, message)
+	_, matches = verify.ExpectRegexp(`
+Signature verification successful!
+Recovered public key: [0-9a-f]+
+Recovered address: (0x[0-9a-fA-F]{40})
+`)
+	recovered := matches[1]
+	verify.ExpectExit()
+
+	if recovered != address {
+		t.Error("recovered address doesn't match generated key")
+	}
+}

cmd/ethkey/run_test.go

@@ -0,0 +1,54 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+type testEthkey struct {
+	*cmdtest.TestCmd
+}
+
+// spawns ethkey with the given command line args.
+func runEthkey(t *testing.T, args ...string) *testEthkey {
+	tt := new(testEthkey)
+	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
+	tt.Run("ethkey-test", args...)
+	return tt
+}
+
+func TestMain(m *testing.M) {
+	// Run the app if we've been exec'd as "ethkey-test" in runEthkey.
+	reexec.Register("ethkey-test", func() {
+		if err := app.Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+	// check if we have been reexec'd
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}

cmd/evm/runner.go

@@ -96,7 +96,9 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
-		_, statedb = gen.ToBlock()
+		db, _ := ethdb.NewMemDatabase()
+		genesis := gen.ToBlock(db)
+		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
 	} else {
 		db, _ := ethdb.NewMemDatabase()

cmd/faucet/faucet.go

@@ -223,7 +223,6 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 			NoDiscovery:      true,
 			DiscoveryV5:      true,
 			ListenAddr:       fmt.Sprintf(":%d", port),
-			DiscoveryV5Addr:  fmt.Sprintf(":%d", port+1),
 			MaxPeers:         25,
 			BootstrapNodesV5: enodes,
 		},

cmd/geth/chaincmd.go

@@ -202,7 +202,7 @@ func importChain(ctx *cli.Context) error {
 
 	if len(ctx.Args()) == 1 {
 		if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
-			utils.Fatalf("Import error: %v", err)
+			log.Error("Import error", "err", err)
 		}
 	} else {
 		for _, arg := range ctx.Args() {
@@ -211,7 +211,7 @@ func importChain(ctx *cli.Context) error {
 			}
 		}
 	}
-
+	chain.Stop()
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing

cmd/geth/config.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bufio"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -29,7 +28,6 @@ import (
 	cli "gopkg.in/urfave/cli.v1"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/contracts/release"
 	"github.com/ethereum/go-ethereum/dashboard"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/node"
@@ -177,21 +175,6 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 	if cfg.Ethstats.URL != "" {
 		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
 	}
-
-	// Add the release oracle service so it boots along with node.
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		config := release.Config{
-			Oracle: relOracle,
-			Major:  uint32(params.VersionMajor),
-			Minor:  uint32(params.VersionMinor),
-			Patch:  uint32(params.VersionPatch),
-		}
-		commit, _ := hex.DecodeString(gitCommit)
-		copy(config.Commit[:], commit)
-		return release.NewReleaseService(ctx, config)
-	}); err != nil {
-		utils.Fatalf("Failed to register the Geth release oracle service: %v", err)
-	}
 	return stack
 }
 

cmd/geth/main.go

@@ -85,10 +85,13 @@ var (
 		utils.FastSyncFlag,
 		utils.LightModeFlag,
 		utils.SyncModeFlag,
+		utils.GCModeFlag,
 		utils.LightServFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
 		utils.CacheFlag,
+		utils.CacheDatabaseFlag,
+		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,

cmd/geth/usage.go

@@ -22,10 +22,11 @@ import (
 	"io"
 	"sort"
 
+	"strings"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"gopkg.in/urfave/cli.v1"
-	"strings"
 )
 
 // AppHelpTemplate is the test template for the default, global app help topic.
@@ -74,6 +75,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.TestnetFlag,
 			utils.RinkebyFlag,
 			utils.SyncModeFlag,
+			utils.GCModeFlag,
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
 			utils.LightServFlag,
@@ -127,6 +129,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "PERFORMANCE TUNING",
 		Flags: []cli.Flag{
 			utils.CacheFlag,
+			utils.CacheDatabaseFlag,
+			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},
 	},

cmd/swarm/config.go

@@ -57,20 +57,24 @@ var (
 
 //constants for environment variables
 const (
-	SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
+	SWARM_ENV_CHEQUEBOOK_ADDR      = "SWARM_CHEQUEBOOK_ADDR"
-	SWARM_ENV_ACCOUNT         = "SWARM_ACCOUNT"
+	SWARM_ENV_ACCOUNT              = "SWARM_ACCOUNT"
-	SWARM_ENV_LISTEN_ADDR     = "SWARM_LISTEN_ADDR"
+	SWARM_ENV_LISTEN_ADDR          = "SWARM_LISTEN_ADDR"
-	SWARM_ENV_PORT            = "SWARM_PORT"
+	SWARM_ENV_PORT                 = "SWARM_PORT"
-	SWARM_ENV_NETWORK_ID      = "SWARM_NETWORK_ID"
+	SWARM_ENV_NETWORK_ID           = "SWARM_NETWORK_ID"
-	SWARM_ENV_SWAP_ENABLE     = "SWARM_SWAP_ENABLE"
+	SWARM_ENV_SWAP_ENABLE          = "SWARM_SWAP_ENABLE"
-	SWARM_ENV_SWAP_API        = "SWARM_SWAP_API"
+	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
-	SWARM_ENV_SYNC_ENABLE     = "SWARM_SYNC_ENABLE"
+	SWARM_ENV_SYNC_ENABLE          = "SWARM_SYNC_ENABLE"
-	SWARM_ENV_ENS_API         = "SWARM_ENS_API"
+	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
-	SWARM_ENV_ENS_ADDR        = "SWARM_ENS_ADDR"
+	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
-	SWARM_ENV_CORS            = "SWARM_CORS"
+	SWARM_ENV_CORS                 = "SWARM_CORS"
-	SWARM_ENV_BOOTNODES       = "SWARM_BOOTNODES"
+	SWARM_ENV_BOOTNODES            = "SWARM_BOOTNODES"
-	SWARM_ENV_PSS_ENABLE      = "SWARM_PSS_ENABLE"
+	SWARM_ENV_PSS_ENABLE           = "SWARM_PSS_ENABLE"
-	GETH_ENV_DATADIR          = "GETH_DATADIR"
+	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
+	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
+	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
+	SWARM_ENV_STORE_RADIUS         = "SWARM_STORE_RADIUS"
+	GETH_ENV_DATADIR               = "GETH_DATADIR"
 )
 
 // These settings ensure that TOML keys use the same names as Go struct fields.
@@ -216,6 +220,22 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.PssEnabled = true
 	}
 
+	if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
+		currentConfig.StoreParams.ChunkDbPath = storePath
+	}
+
+	if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
+		currentConfig.StoreParams.DbCapacity = storeCapacity
+	}
+
+	if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
+		currentConfig.StoreParams.CacheCapacity = storeCacheCapacity
+	}
+
+	if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 {
+		currentConfig.StoreParams.Radius = storeRadius
+	}
+
 	return currentConfig
 
 }

cmd/swarm/main.go

@@ -154,6 +154,26 @@ var (
 		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
 		EnvVar: SWARM_ENV_CORS,
 	}
+	SwarmStorePath = cli.StringFlag{
+		Name:   "store.path",
+		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
+		EnvVar: SWARM_ENV_STORE_PATH,
+	}
+	SwarmStoreCapacity = cli.Uint64Flag{
+		Name:   "store.size",
+		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
+		EnvVar: SWARM_ENV_STORE_CAPACITY,
+	}
+	SwarmStoreCacheCapacity = cli.UintFlag{
+		Name:   "store.cache.size",
+		Usage:  "Number of recent chunks cached in memory (default 5000)",
+		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
+	}
+	SwarmStoreRadius = cli.IntFlag{
+		Name:   "store.radius",
+		Usage:  "Minimum proximity order (number of identical prefix bits of address key) for chunks to warrant storage (default 0)",
+		EnvVar: SWARM_ENV_STORE_RADIUS,
+	}
 
 	// the following flags are deprecated and should be removed in the future
 	DeprecatedEthAPIFlag = cli.StringFlag{
@@ -367,6 +387,11 @@ DEPRECATED: use 'swarm db clean'.
 		SwarmUploadMimeType,
 		// pss flags
 		SwarmPssEnabledFlag,
+		// storage flags
+		SwarmStorePath,
+		SwarmStoreCapacity,
+		SwarmStoreCacheCapacity,
+		SwarmStoreRadius,
 		//deprecated flags
 		DeprecatedEthAPIFlag,
 	}
@@ -535,7 +560,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 		}
 
 		// In production, mockStore must be always nil.
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil)
+		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, nil)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

cmd/utils/cmd.go

@@ -116,7 +116,6 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 			return err
 		}
 	}
-
 	stream := rlp.NewStream(reader, 0)
 
 	// Run actual the import.
@@ -150,25 +149,34 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 		if checkInterrupt() {
 			return fmt.Errorf("interrupted")
 		}
-		if hasAllBlocks(chain, blocks[:i]) {
+		missing := missingBlocks(chain, blocks[:i])
+		if len(missing) == 0 {
 			log.Info("Skipping batch as all blocks present", "batch", batch, "first", blocks[0].Hash(), "last", blocks[i-1].Hash())
 			continue
 		}
-
+		if _, err := chain.InsertChain(missing); err != nil {
-		if _, err := chain.InsertChain(blocks[:i]); err != nil {
 			return fmt.Errorf("invalid block %d: %v", n, err)
 		}
 	}
 	return nil
 }
 
-func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool {
+func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block {
-	for _, b := range bs {
+	head := chain.CurrentBlock()
-		if !chain.HasBlock(b.Hash(), b.NumberU64()) {
+	for i, block := range blocks {
-			return false
+		// If we're behind the chain head, only check block, state is available at head
+		if head.NumberU64() > block.NumberU64() {
+			if !chain.HasBlock(block.Hash(), block.NumberU64()) {
+				return blocks[i:]
+			}
+			continue
+		}
+		// If we're above the chain head, state availability is a must
+		if !chain.HasBlockAndState(block.Hash(), block.NumberU64()) {
+			return blocks[i:]
 		}
 	}
-	return true
+	return nil
 }
 
 func ExportChain(blockchain *core.BlockChain, fn string) error {

cmd/utils/flags.go

@@ -170,7 +170,11 @@ var (
 		Usage: `Blockchain sync mode ("fast", "full", or "light")`,
 		Value: &defaultSyncMode,
 	}
-
+	GCModeFlag = cli.StringFlag{
+		Name:  "gcmode",
+		Usage: `Blockchain garbage collection mode ("full", "archive")`,
+		Value: "full",
+	}
 	LightServFlag = cli.IntFlag{
 		Name:  "lightserv",
 		Usage: "Maximum percentage of time allowed for serving LES requests (0-90)",
@@ -179,7 +183,7 @@ var (
 	LightPeersFlag = cli.IntFlag{
 		Name:  "lightpeers",
 		Usage: "Maximum number of LES client peers",
-		Value: 20,
+		Value: eth.DefaultConfig.LightPeers,
 	}
 	LightKDFFlag = cli.BoolFlag{
 		Name:  "lightkdf",
@@ -293,8 +297,18 @@ var (
 	// Performance tuning settings
 	CacheFlag = cli.IntFlag{
 		Name:  "cache",
-		Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)",
+		Usage: "Megabytes of memory allocated to internal caching",
-		Value: 128,
+		Value: 1024,
+	}
+	CacheDatabaseFlag = cli.IntFlag{
+		Name:  "cache.database",
+		Usage: "Percentage of cache memory allowance to use for database io",
+		Value: 75,
+	}
+	CacheGCFlag = cli.IntFlag{
+		Name:  "cache.gc",
+		Usage: "Percentage of cache memory allowance to use for trie pruning",
+		Value: 25,
 	}
 	TrieCacheGenFlag = cli.IntFlag{
 		Name:  "trie-cache-gens",
@@ -612,7 +626,7 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
 			urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
 		}
 	case ctx.GlobalBool(RinkebyFlag.Name):
-		urls = params.RinkebyV5Bootnodes
+		urls = params.RinkebyBootnodes
 	case cfg.BootstrapNodesV5 != nil:
 		return // already set, don't apply defaults.
 	}
@@ -636,14 +650,6 @@ func setListenAddress(ctx *cli.Context, cfg *p2p.Config) {
 	}
 }
 
-// setDiscoveryV5Address creates a UDP listening address string from set command
-// line flags for the V5 discovery protocol.
-func setDiscoveryV5Address(ctx *cli.Context, cfg *p2p.Config) {
-	if ctx.GlobalIsSet(ListenPortFlag.Name) {
-		cfg.DiscoveryV5Addr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)+1)
-	}
-}
-
 // setNAT creates a port mapper from command line flags.
 func setNAT(ctx *cli.Context, cfg *p2p.Config) {
 	if ctx.GlobalIsSet(NATFlag.Name) {
@@ -722,13 +728,15 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
 // makeDatabaseHandles raises out the number of allowed file handles per process
 // for Geth and returns half of the allowance to assign to the database.
 func makeDatabaseHandles() int {
-	if err := fdlimit.Raise(2048); err != nil {
-		Fatalf("Failed to raise file descriptor allowance: %v", err)
-	}
 	limit, err := fdlimit.Current()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
+	if limit < 2048 {
+		if err := fdlimit.Raise(2048); err != nil {
+			Fatalf("Failed to raise file descriptor allowance: %v", err)
+		}
+	}
 	if limit > 2048 { // cap database file descriptors even if more is available
 		limit = 2048
 	}
@@ -794,24 +802,43 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 	setNodeKey(ctx, cfg)
 	setNAT(ctx, cfg)
 	setListenAddress(ctx, cfg)
-	setDiscoveryV5Address(ctx, cfg)
 	setBootstrapNodes(ctx, cfg)
 	setBootstrapNodesV5(ctx, cfg)
 
+	lightClient := ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalString(SyncModeFlag.Name) == "light"
+	lightServer := ctx.GlobalInt(LightServFlag.Name) != 0
+	lightPeers := ctx.GlobalInt(LightPeersFlag.Name)
+
 	if ctx.GlobalIsSet(MaxPeersFlag.Name) {
 		cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
+	} else {
+		if lightServer {
+			cfg.MaxPeers += lightPeers
+		}
+		if lightClient && ctx.GlobalIsSet(LightPeersFlag.Name) && cfg.MaxPeers < lightPeers {
+			cfg.MaxPeers = lightPeers
+		}
 	}
+	if !(lightClient || lightServer) {
+		lightPeers = 0
+	}
+	ethPeers := cfg.MaxPeers - lightPeers
+	if lightClient {
+		ethPeers = 0
+	}
+	log.Info("Maximum peer count", "ETH", ethPeers, "LES", lightPeers, "total", cfg.MaxPeers)
+
 	if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) {
 		cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name)
 	}
-	if ctx.GlobalIsSet(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name) {
+	if ctx.GlobalIsSet(NoDiscoverFlag.Name) || lightClient {
 		cfg.NoDiscovery = true
 	}
 
 	// if we're running a light client or server, force enable the v5 peer discovery
 	// unless it is explicitly disabled with --nodiscover note that explicitly specifying
 	// --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
-	forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name)
+	forceV5Discovery := (lightClient || lightServer) && !ctx.GlobalBool(NoDiscoverFlag.Name)
 	if ctx.GlobalIsSet(DiscoveryV5Flag.Name) {
 		cfg.DiscoveryV5 = ctx.GlobalBool(DiscoveryV5Flag.Name)
 	} else if forceV5Discovery {
@@ -830,7 +857,6 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 		// --dev mode can't use p2p networking.
 		cfg.MaxPeers = 0
 		cfg.ListenAddr = ":0"
-		cfg.DiscoveryV5Addr = ":0"
 		cfg.NoDiscovery = true
 		cfg.DiscoveryV5 = false
 	}
@@ -1009,11 +1035,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 		cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
 	}
 
-	if ctx.GlobalIsSet(CacheFlag.Name) {
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
-		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name)
+		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 	}
 	cfg.DatabaseHandles = makeDatabaseHandles()
 
+	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
+		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
+	}
+	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
+
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
+		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
 		cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
 	}
@@ -1145,7 +1179,7 @@ func SetupNetwork(ctx *cli.Context) {
 // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	var (
-		cache   = ctx.GlobalInt(CacheFlag.Name)
+		cache   = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 		handles = makeDatabaseHandles()
 	)
 	name := "chaindata"
@@ -1197,8 +1231,19 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 			})
 		}
 	}
+	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
+		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
+	}
+	cache := &core.CacheConfig{
+		Disabled:      ctx.GlobalString(GCModeFlag.Name) == "archive",
+		TrieNodeLimit: eth.DefaultConfig.TrieCache,
+		TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
+	}
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
+		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
-	chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
+	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}

cmd/wnode/main.go

@@ -601,7 +601,7 @@ func requestExpiredMessagesLoop() {
 	if err != nil {
 		utils.Fatalf("Failed to save symmetric key for mail request: %s", err)
 	}
-	peerID = extractIdFromEnode(*argEnode)
+	peerID = extractIDFromEnode(*argEnode)
 	shh.AllowP2PMessagesFromPeer(peerID)
 
 	for {
@@ -652,7 +652,7 @@ func requestExpiredMessagesLoop() {
 	}
 }
 
-func extractIdFromEnode(s string) []byte {
+func extractIDFromEnode(s string) []byte {
 	n, err := discover.ParseNode(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)

common/size.go

@@ -20,18 +20,29 @@ import (
 	"fmt"
 )
 
+// StorageSize is a wrapper around a float value that supports user friendly
+// formatting.
 type StorageSize float64
 
-func (self StorageSize) String() string {
+// String implements the stringer interface.
-	if self > 1000000 {
+func (s StorageSize) String() string {
-		return fmt.Sprintf("%.2f mB", self/1000000)
+	if s > 1000000 {
-	} else if self > 1000 {
+		return fmt.Sprintf("%.2f mB", s/1000000)
-		return fmt.Sprintf("%.2f kB", self/1000)
+	} else if s > 1000 {
+		return fmt.Sprintf("%.2f kB", s/1000)
 	} else {
-		return fmt.Sprintf("%.2f B", self)
+		return fmt.Sprintf("%.2f B", s)
 	}
 }
 
-func (self StorageSize) Int64() int64 {
+// TerminalString implements log.TerminalStringer, formatting a string for console
-	return int64(self)
+// output during logging.
+func (s StorageSize) TerminalString() string {
+	if s > 1000000 {
+		return fmt.Sprintf("%.2fmB", s/1000000)
+	} else if s > 1000 {
+		return fmt.Sprintf("%.2fkB", s/1000)
+	} else {
+		return fmt.Sprintf("%.2fB", s)
+	}
 }

consensus/errors.go

@@ -23,6 +23,10 @@ var (
 	// that is unknown.
 	ErrUnknownAncestor = errors.New("unknown ancestor")
 
+	// ErrPrunedAncestor is returned when validating a block requires an ancestor
+	// that is known, but the state of which is not available.
+	ErrPrunedAncestor = errors.New("pruned ancestor")
+
 	// ErrFutureBlock is returned when a block's timestamp is in the future according
 	// to the current node.
 	ErrFutureBlock = errors.New("block in the future")

consensus/ethash/algorithm.go

@@ -355,9 +355,11 @@ func hashimotoFull(dataset []uint32, hash []byte, nonce uint64) ([]byte, []byte)
 	return hashimoto(hash, nonce, uint64(len(dataset))*4, lookup)
 }
 
+const maxEpoch = 2048
+
 // datasetSizes is a lookup table for the ethash dataset size for the first 2048
 // epochs (i.e. 61440000 blocks).
-var datasetSizes = []uint64{
+var datasetSizes = [maxEpoch]uint64{
 	1073739904, 1082130304, 1090514816, 1098906752, 1107293056,
 	1115684224, 1124070016, 1132461952, 1140849536, 1149232768,
 	1157627776, 1166013824, 1174404736, 1182786944, 1191180416,
@@ -771,7 +773,7 @@ var datasetSizes = []uint64{
 
 // cacheSizes is a lookup table for the ethash verification cache size for the
 // first 2048 epochs (i.e. 61440000 blocks).
-var cacheSizes = []uint64{
+var cacheSizes = [maxEpoch]uint64{
 	16776896, 16907456, 17039296, 17170112, 17301056, 17432512, 17563072,
 	17693888, 17824192, 17955904, 18087488, 18218176, 18349504, 18481088,
 	18611392, 18742336, 18874304, 19004224, 19135936, 19267264, 19398208,

consensus/ethash/algorithm_go1.7.go

@@ -25,7 +25,7 @@ package ethash
 func cacheSize(block uint64) uint64 {
 	// If we have a pre-generated value, use that
 	epoch := int(block / epochLength)
-	if epoch < len(cacheSizes) {
+	if epoch < maxEpoch {
 		return cacheSizes[epoch]
 	}
 	// We don't have a way to verify primes fast before Go 1.8
@@ -39,7 +39,7 @@ func cacheSize(block uint64) uint64 {
 func datasetSize(block uint64) uint64 {
 	// If we have a pre-generated value, use that
 	epoch := int(block / epochLength)
-	if epoch < len(datasetSizes) {
+	if epoch < maxEpoch {
 		return datasetSizes[epoch]
 	}
 	// We don't have a way to verify primes fast before Go 1.8

consensus/ethash/algorithm_go1.8.go

@@ -20,17 +20,20 @@ package ethash
 
 import "math/big"
 
-// cacheSize calculates and returns the size of the ethash verification cache that
+// cacheSize returns the size of the ethash verification cache that belongs to a certain
-// belongs to a certain block number. The cache size grows linearly, however, we
+// block number.
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
 func cacheSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
 	epoch := int(block / epochLength)
-	if epoch < len(cacheSizes) {
+	if epoch < maxEpoch {
 		return cacheSizes[epoch]
 	}
-	// No known cache size, calculate manually (sanity branch only)
+	return calcCacheSize(epoch)
+}
+
+// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcCacheSize(epoch int) uint64 {
 	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
 	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * hashBytes
@@ -38,17 +41,20 @@ func cacheSize(block uint64) uint64 {
 	return size
 }
 
-// datasetSize calculates and returns the size of the ethash mining dataset that
+// datasetSize returns the size of the ethash mining dataset that belongs to a certain
-// belongs to a certain block number. The dataset size grows linearly, however, we
+// block number.
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
 func datasetSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
 	epoch := int(block / epochLength)
-	if epoch < len(datasetSizes) {
+	if epoch < maxEpoch {
 		return datasetSizes[epoch]
 	}
-	// No known dataset size, calculate manually (sanity branch only)
+	return calcDatasetSize(epoch)
+}
+
+// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcDatasetSize(epoch int) uint64 {
 	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
 	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * mixBytes

consensus/ethash/algorithm_go1.8_test.go

@@ -23,24 +23,15 @@ import "testing"
 // Tests whether the dataset size calculator works correctly by cross checking the
 // hard coded lookup table with the value generated by it.
 func TestSizeCalculations(t *testing.T) {
-	var tests []uint64
+	// Verify all the cache and dataset sizes from the lookup table.
-
+	for epoch, want := range cacheSizes {
-	// Verify all the cache sizes from the lookup table
+		if size := calcCacheSize(epoch); size != want {
-	defer func(sizes []uint64) { cacheSizes = sizes }(cacheSizes)
+			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
-	tests, cacheSizes = cacheSizes, []uint64{}
-
-	for i, test := range tests {
-		if size := cacheSize(uint64(i*epochLength) + 1); size != test {
-			t.Errorf("cache %d: cache size mismatch: have %d, want %d", i, size, test)
 		}
 	}
-	// Verify all the dataset sizes from the lookup table
+	for epoch, want := range datasetSizes {
-	defer func(sizes []uint64) { datasetSizes = sizes }(datasetSizes)
+		if size := calcDatasetSize(epoch); size != want {
-	tests, datasetSizes = datasetSizes, []uint64{}
+			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
-
-	for i, test := range tests {
-		if size := datasetSize(uint64(i*epochLength) + 1); size != test {
-			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", i, size, test)
 		}
 	}
 }

consensus/ethash/consensus.go

@@ -476,7 +476,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	}
 	// Sanity check that the block number is below the lookup table size (60M blocks)
 	number := header.Number.Uint64()
-	if number/epochLength >= uint64(len(cacheSizes)) {
+	if number/epochLength >= maxEpoch {
 		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
 		return errNonceOutOfRange
 	}
@@ -484,14 +484,18 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	if header.Difficulty.Sign() <= 0 {
 		return errInvalidDifficulty
 	}
+
 	// Recompute the digest and PoW value and verify against the header
 	cache := ethash.cache(number)
-
 	size := datasetSize(number)
 	if ethash.config.PowMode == ModeTest {
 		size = 32 * 1024
 	}
-	digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+	digest, result := hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+	// Caches are unmapped in a finalizer. Ensure that the cache stays live
+	// until after the call to hashimotoLight so it's not unmapped while being used.
+	runtime.KeepAlive(cache)
+
 	if !bytes.Equal(header.MixDigest[:], digest) {
 		return errInvalidMixDigest
 	}

consensus/ethash/ethash.go

@@ -26,6 +26,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strconv"
 	"sync"
 	"time"
@@ -35,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/hashicorp/golang-lru/simplelru"
 	metrics "github.com/rcrowley/go-metrics"
 )
 
@@ -142,32 +144,82 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint
 	return memoryMap(path)
 }
 
+// lru tracks caches or datasets by their last use time, keeping at most N of them.
+type lru struct {
+	what string
+	new  func(epoch uint64) interface{}
+	mu   sync.Mutex
+	// Items are kept in a LRU cache, but there is a special case:
+	// We always keep an item for (highest seen epoch) + 1 as the 'future item'.
+	cache      *simplelru.LRU
+	future     uint64
+	futureItem interface{}
+}
+
+// newlru create a new least-recently-used cache for ither the verification caches
+// or the mining datasets.
+func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru {
+	if maxItems <= 0 {
+		maxItems = 1
+	}
+	cache, _ := simplelru.NewLRU(maxItems, func(key, value interface{}) {
+		log.Trace("Evicted ethash "+what, "epoch", key)
+	})
+	return &lru{what: what, new: new, cache: cache}
+}
+
+// get retrieves or creates an item for the given epoch. The first return value is always
+// non-nil. The second return value is non-nil if lru thinks that an item will be useful in
+// the near future.
+func (lru *lru) get(epoch uint64) (item, future interface{}) {
+	lru.mu.Lock()
+	defer lru.mu.Unlock()
+
+	// Get or create the item for the requested epoch.
+	item, ok := lru.cache.Get(epoch)
+	if !ok {
+		if lru.future > 0 && lru.future == epoch {
+			item = lru.futureItem
+		} else {
+			log.Trace("Requiring new ethash "+lru.what, "epoch", epoch)
+			item = lru.new(epoch)
+		}
+		lru.cache.Add(epoch, item)
+	}
+	// Update the 'future item' if epoch is larger than previously seen.
+	if epoch < maxEpoch-1 && lru.future < epoch+1 {
+		log.Trace("Requiring new future ethash "+lru.what, "epoch", epoch+1)
+		future = lru.new(epoch + 1)
+		lru.future = epoch + 1
+		lru.futureItem = future
+	}
+	return item, future
+}
+
 // cache wraps an ethash cache with some metadata to allow easier concurrent use.
 type cache struct {
-	epoch uint64 // Epoch for which this cache is relevant
+	epoch uint64    // Epoch for which this cache is relevant
+	dump  *os.File  // File descriptor of the memory mapped cache
+	mmap  mmap.MMap // Memory map itself to unmap before releasing
+	cache []uint32  // The actual cache data content (may be memory mapped)
+	once  sync.Once // Ensures the cache is generated only once
+}
 
-	dump *os.File  // File descriptor of the memory mapped cache
+// newCache creates a new ethash verification cache and returns it as a plain Go
-	mmap mmap.MMap // Memory map itself to unmap before releasing
+// interface to be usable in an LRU cache.
-
+func newCache(epoch uint64) interface{} {
-	cache []uint32   // The actual cache data content (may be memory mapped)
+	return &cache{epoch: epoch}
-	used  time.Time  // Timestamp of the last use for smarter eviction
-	once  sync.Once  // Ensures the cache is generated only once
-	lock  sync.Mutex // Ensures thread safety for updating the usage time
 }
 
 // generate ensures that the cache content is generated before use.
 func (c *cache) generate(dir string, limit int, test bool) {
 	c.once.Do(func() {
-		// If we have a testing cache, generate and return
-		if test {
-			c.cache = make([]uint32, 1024/4)
-			generateCache(c.cache, c.epoch, seedHash(c.epoch*epochLength+1))
-			return
-		}
-		// If we don't store anything on disk, generate and return
 		size := cacheSize(c.epoch*epochLength + 1)
 		seed := seedHash(c.epoch*epochLength + 1)
-
+		if test {
+			size = 1024
+		}
+		// If we don't store anything on disk, generate and return.
 		if dir == "" {
 			c.cache = make([]uint32, size/4)
 			generateCache(c.cache, c.epoch, seed)
@@ -181,6 +233,10 @@ func (c *cache) generate(dir string, limit int, test bool) {
 		path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian))
 		logger := log.New("epoch", c.epoch)
 
+		// We're about to mmap the file, ensure that the mapping is cleaned up when the
+		// cache becomes unused.
+		runtime.SetFinalizer(c, (*cache).finalizer)
+
 		// Try to load the file from disk and memory map it
 		var err error
 		c.dump, c.mmap, c.cache, err = memoryMap(path)
@@ -207,49 +263,41 @@ func (c *cache) generate(dir string, limit int, test bool) {
 	})
 }
 
-// release closes any file handlers and memory maps open.
+// finalizer unmaps the memory and closes the file.
-func (c *cache) release() {
+func (c *cache) finalizer() {
 	if c.mmap != nil {
 		c.mmap.Unmap()
-		c.mmap = nil
-	}
-	if c.dump != nil {
 		c.dump.Close()
-		c.dump = nil
+		c.mmap, c.dump = nil, nil
 	}
 }
 
 // dataset wraps an ethash dataset with some metadata to allow easier concurrent use.
 type dataset struct {
-	epoch uint64 // Epoch for which this cache is relevant
+	epoch   uint64    // Epoch for which this cache is relevant
+	dump    *os.File  // File descriptor of the memory mapped cache
+	mmap    mmap.MMap // Memory map itself to unmap before releasing
+	dataset []uint32  // The actual cache data content
+	once    sync.Once // Ensures the cache is generated only once
+}
 
-	dump *os.File  // File descriptor of the memory mapped cache
+// newDataset creates a new ethash mining dataset and returns it as a plain Go
-	mmap mmap.MMap // Memory map itself to unmap before releasing
+// interface to be usable in an LRU cache.
-
+func newDataset(epoch uint64) interface{} {
-	dataset []uint32   // The actual cache data content
+	return &dataset{epoch: epoch}
-	used    time.Time  // Timestamp of the last use for smarter eviction
-	once    sync.Once  // Ensures the cache is generated only once
-	lock    sync.Mutex // Ensures thread safety for updating the usage time
 }
 
 // generate ensures that the dataset content is generated before use.
 func (d *dataset) generate(dir string, limit int, test bool) {
 	d.once.Do(func() {
-		// If we have a testing dataset, generate and return
-		if test {
-			cache := make([]uint32, 1024/4)
-			generateCache(cache, d.epoch, seedHash(d.epoch*epochLength+1))
-
-			d.dataset = make([]uint32, 32*1024/4)
-			generateDataset(d.dataset, d.epoch, cache)
-
-			return
-		}
-		// If we don't store anything on disk, generate and return
 		csize := cacheSize(d.epoch*epochLength + 1)
 		dsize := datasetSize(d.epoch*epochLength + 1)
 		seed := seedHash(d.epoch*epochLength + 1)
-
+		if test {
+			csize = 1024
+			dsize = 32 * 1024
+		}
+		// If we don't store anything on disk, generate and return
 		if dir == "" {
 			cache := make([]uint32, csize/4)
 			generateCache(cache, d.epoch, seed)
@@ -265,6 +313,10 @@ func (d *dataset) generate(dir string, limit int, test bool) {
 		path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian))
 		logger := log.New("epoch", d.epoch)
 
+		// We're about to mmap the file, ensure that the mapping is cleaned up when the
+		// cache becomes unused.
+		runtime.SetFinalizer(d, (*dataset).finalizer)
+
 		// Try to load the file from disk and memory map it
 		var err error
 		d.dump, d.mmap, d.dataset, err = memoryMap(path)
@@ -294,15 +346,12 @@ func (d *dataset) generate(dir string, limit int, test bool) {
 	})
 }
 
-// release closes any file handlers and memory maps open.
+// finalizer closes any file handlers and memory maps open.
-func (d *dataset) release() {
+func (d *dataset) finalizer() {
 	if d.mmap != nil {
 		d.mmap.Unmap()
-		d.mmap = nil
-	}
-	if d.dump != nil {
 		d.dump.Close()
-		d.dump = nil
+		d.mmap, d.dump = nil, nil
 	}
 }
 
@@ -310,14 +359,12 @@ func (d *dataset) release() {
 func MakeCache(block uint64, dir string) {
 	c := cache{epoch: block / epochLength}
 	c.generate(dir, math.MaxInt32, false)
-	c.release()
 }
 
 // MakeDataset generates a new ethash dataset and optionally stores it to disk.
 func MakeDataset(block uint64, dir string) {
 	d := dataset{epoch: block / epochLength}
 	d.generate(dir, math.MaxInt32, false)
-	d.release()
 }
 
 // Mode defines the type and amount of PoW verification an ethash engine makes.
@@ -347,10 +394,8 @@ type Config struct {
 type Ethash struct {
 	config Config
 
-	caches   map[uint64]*cache   // In memory caches to avoid regenerating too often
+	caches   *lru // In memory caches to avoid regenerating too often
-	fcache   *cache              // Pre-generated cache for the estimated future epoch
+	datasets *lru // In memory datasets to avoid regenerating too often
-	datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often
-	fdataset *dataset            // Pre-generated dataset for the estimated future epoch
 
 	// Mining related fields
 	rand     *rand.Rand    // Properly seeded random source for nonces
@@ -380,8 +425,8 @@ func New(config Config) *Ethash {
 	}
 	return &Ethash{
 		config:   config,
-		caches:   make(map[uint64]*cache),
+		caches:   newlru("cache", config.CachesInMem, newCache),
-		datasets: make(map[uint64]*dataset),
+		datasets: newlru("dataset", config.DatasetsInMem, newDataset),
 		update:   make(chan struct{}),
 		hashrate: metrics.NewMeter(),
 	}
@@ -390,16 +435,7 @@ func New(config Config) *Ethash {
 // NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
 func NewTester() *Ethash {
-	return &Ethash{
+	return New(Config{CachesInMem: 1, PowMode: ModeTest})
-		config: Config{
-			CachesInMem: 1,
-			PowMode:     ModeTest,
-		},
-		caches:   make(map[uint64]*cache),
-		datasets: make(map[uint64]*dataset),
-		update:   make(chan struct{}),
-		hashrate: metrics.NewMeter(),
-	}
 }
 
 // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts
@@ -456,126 +492,40 @@ func NewShared() *Ethash {
 // cache tries to retrieve a verification cache for the specified block number
 // by first checking against a list of in-memory caches, then against caches
 // stored on disk, and finally generating one if none can be found.
-func (ethash *Ethash) cache(block uint64) []uint32 {
+func (ethash *Ethash) cache(block uint64) *cache {
 	epoch := block / epochLength
+	currentI, futureI := ethash.caches.get(epoch)
+	current := currentI.(*cache)
 
-	// If we have a PoW for that epoch, use that
+	// Wait for generation finish.
-	ethash.lock.Lock()
-
-	current, future := ethash.caches[epoch], (*cache)(nil)
-	if current == nil {
-		// No in-memory cache, evict the oldest if the cache limit was reached
-		for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.config.CachesInMem {
-			var evict *cache
-			for _, cache := range ethash.caches {
-				if evict == nil || evict.used.After(cache.used) {
-					evict = cache
-				}
-			}
-			delete(ethash.caches, evict.epoch)
-			evict.release()
-
-			log.Trace("Evicted ethash cache", "epoch", evict.epoch, "used", evict.used)
-		}
-		// If we have the new cache pre-generated, use that, otherwise create a new one
-		if ethash.fcache != nil && ethash.fcache.epoch == epoch {
-			log.Trace("Using pre-generated cache", "epoch", epoch)
-			current, ethash.fcache = ethash.fcache, nil
-		} else {
-			log.Trace("Requiring new ethash cache", "epoch", epoch)
-			current = &cache{epoch: epoch}
-		}
-		ethash.caches[epoch] = current
-
-		// If we just used up the future cache, or need a refresh, regenerate
-		if ethash.fcache == nil || ethash.fcache.epoch <= epoch {
-			if ethash.fcache != nil {
-				ethash.fcache.release()
-			}
-			log.Trace("Requiring new future ethash cache", "epoch", epoch+1)
-			future = &cache{epoch: epoch + 1}
-			ethash.fcache = future
-		}
-		// New current cache, set its initial timestamp
-		current.used = time.Now()
-	}
-	ethash.lock.Unlock()
-
-	// Wait for generation finish, bump the timestamp and finalize the cache
 	current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
 
-	current.lock.Lock()
+	// If we need a new future cache, now's a good time to regenerate it.
-	current.used = time.Now()
+	if futureI != nil {
-	current.lock.Unlock()
+		future := futureI.(*cache)
-
-	// If we exhausted the future cache, now's a good time to regenerate it
-	if future != nil {
 		go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
 	}
-	return current.cache
+	return current
 }
 
 // dataset tries to retrieve a mining dataset for the specified block number
 // by first checking against a list of in-memory datasets, then against DAGs
 // stored on disk, and finally generating one if none can be found.
-func (ethash *Ethash) dataset(block uint64) []uint32 {
+func (ethash *Ethash) dataset(block uint64) *dataset {
 	epoch := block / epochLength
+	currentI, futureI := ethash.datasets.get(epoch)
+	current := currentI.(*dataset)
 
-	// If we have a PoW for that epoch, use that
+	// Wait for generation finish.
-	ethash.lock.Lock()
-
-	current, future := ethash.datasets[epoch], (*dataset)(nil)
-	if current == nil {
-		// No in-memory dataset, evict the oldest if the dataset limit was reached
-		for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.config.DatasetsInMem {
-			var evict *dataset
-			for _, dataset := range ethash.datasets {
-				if evict == nil || evict.used.After(dataset.used) {
-					evict = dataset
-				}
-			}
-			delete(ethash.datasets, evict.epoch)
-			evict.release()
-
-			log.Trace("Evicted ethash dataset", "epoch", evict.epoch, "used", evict.used)
-		}
-		// If we have the new cache pre-generated, use that, otherwise create a new one
-		if ethash.fdataset != nil && ethash.fdataset.epoch == epoch {
-			log.Trace("Using pre-generated dataset", "epoch", epoch)
-			current = &dataset{epoch: ethash.fdataset.epoch} // Reload from disk
-			ethash.fdataset = nil
-		} else {
-			log.Trace("Requiring new ethash dataset", "epoch", epoch)
-			current = &dataset{epoch: epoch}
-		}
-		ethash.datasets[epoch] = current
-
-		// If we just used up the future dataset, or need a refresh, regenerate
-		if ethash.fdataset == nil || ethash.fdataset.epoch <= epoch {
-			if ethash.fdataset != nil {
-				ethash.fdataset.release()
-			}
-			log.Trace("Requiring new future ethash dataset", "epoch", epoch+1)
-			future = &dataset{epoch: epoch + 1}
-			ethash.fdataset = future
-		}
-		// New current dataset, set its initial timestamp
-		current.used = time.Now()
-	}
-	ethash.lock.Unlock()
-
-	// Wait for generation finish, bump the timestamp and finalize the cache
 	current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
 
-	current.lock.Lock()
+	// If we need a new future dataset, now's a good time to regenerate it.
-	current.used = time.Now()
+	if futureI != nil {
-	current.lock.Unlock()
+		future := futureI.(*dataset)
-
-	// If we exhausted the future dataset, now's a good time to regenerate it
-	if future != nil {
 		go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
 	}
-	return current.dataset
+
+	return current
 }
 
 // Threads returns the number of mining threads currently enabled. This doesn't

consensus/ethash/ethash_test.go

@@ -17,7 +17,11 @@
 package ethash
 
 import (
+	"io/ioutil"
 	"math/big"
+	"math/rand"
+	"os"
+	"sync"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/core/types"
@@ -38,3 +42,38 @@ func TestTestMode(t *testing.T) {
 		t.Fatalf("unexpected verification error: %v", err)
 	}
 }
+
+// This test checks that cache lru logic doesn't crash under load.
+// It reproduces https://github.com/ethereum/go-ethereum/issues/14943
+func TestCacheFileEvict(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "ethash-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest})
+
+	workers := 8
+	epochs := 100
+	var wg sync.WaitGroup
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go verifyTest(&wg, e, i, epochs)
+	}
+	wg.Wait()
+}
+
+func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) {
+	defer wg.Done()
+
+	const wiggle = 4 * epochLength
+	r := rand.New(rand.NewSource(int64(workerIndex)))
+	for epoch := 0; epoch < epochs; epoch++ {
+		block := int64(epoch)*epochLength - wiggle/2 + r.Int63n(wiggle)
+		if block < 0 {
+			block = 0
+		}
+		head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)}
+		e.VerifySeal(nil, head)
+	}
+}

consensus/ethash/sealer.go

@@ -97,10 +97,9 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop
 func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) {
 	// Extract some data from the header
 	var (
-		header = block.Header()
+		header  = block.Header()
-		hash   = header.HashNoNonce().Bytes()
+		hash    = header.HashNoNonce().Bytes()
-		target = new(big.Int).Div(maxUint256, header.Difficulty)
+		target  = new(big.Int).Div(maxUint256, header.Difficulty)
-
 		number  = header.Number.Uint64()
 		dataset = ethash.dataset(number)
 	)
@@ -111,13 +110,14 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s
 	)
 	logger := log.New("miner", id)
 	logger.Trace("Started ethash search for new nonces", "seed", seed)
+search:
 	for {
 		select {
 		case <-abort:
 			// Mining terminated, update stats and abort
 			logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed)
 			ethash.hashrate.Mark(attempts)
-			return
+			break search
 
 		default:
 			// We don't have to update hash rate on every nonce, so update after after 2^X nonces
@@ -127,7 +127,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s
 				attempts = 0
 			}
 			// Compute the PoW value of this nonce
-			digest, result := hashimotoFull(dataset, hash, nonce)
+			digest, result := hashimotoFull(dataset.dataset, hash, nonce)
 			if new(big.Int).SetBytes(result).Cmp(target) <= 0 {
 				// Correct nonce found, create a new header with it
 				header = types.CopyHeader(header)
@@ -141,9 +141,12 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s
 				case <-abort:
 					logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce)
 				}
-				return
+				break search
 			}
 			nonce++
 		}
 	}
+	// Datasets are unmapped in a finalizer. Ensure that the dataset stays live
+	// during sealing so it's not unmapped while being read.
+	runtime.KeepAlive(dataset)
 }

contracts/chequebook/contract/chequebook.go

@@ -7,17 +7,19 @@ import (
 	"math/big"
 	"strings"
 
+	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
 )
 
 // ChequebookABI is the input ABI used to generate the binding from.
 const ChequebookABI = "[{\"constant\":false,\"inputs\":[],\"name\":\"kill\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"name\":\"\",\"type\":\"address\"}],\"name\":\"sent\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"beneficiary\",\"type\":\"address\"},{\"name\":\"amount\",\"type\":\"uint256\"},{\"name\":\"sig_v\",\"type\":\"uint8\"},{\"name\":\"sig_r\",\"type\":\"bytes32\"},{\"name\":\"sig_s\",\"type\":\"bytes32\"}],\"name\":\"cash\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"payable\":true,\"stateMutability\":\"payable\",\"type\":\"fallback\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"name\":\"deadbeat\",\"type\":\"address\"}],\"name\":\"Overdraft\",\"type\":\"event\"}]"
 
 // ChequebookBin is the compiled bytecode used for deploying new contracts.
-const ChequebookBin = `0x606060405260008054600160a060020a033316600160a060020a03199091161790556102ec806100306000396000f3006060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a7230582014e927522ca5cd8f68529ac4d3b9cdf36d40e09d8a33b70008248d1abebf79680029`
+const ChequebookBin = `0x606060405260008054600160a060020a033316600160a060020a03199091161790556102ec806100306000396000f3006060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a72305820533e856fc37e3d64d1706bcc7dfb6b1d490c8d566ea498d9d01ec08965a896ca0029`
 
 // DeployChequebook deploys a new Ethereum contract, binding an instance of Chequebook to it.
 func DeployChequebook(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *Chequebook, error) {
@@ -29,13 +31,14 @@ func DeployChequebook(auth *bind.TransactOpts, backend bind.ContractBackend) (co
 	if err != nil {
 		return common.Address{}, nil, nil, err
 	}
-	return address, tx, &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}}, nil
+	return address, tx, &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}, ChequebookFilterer: ChequebookFilterer{contract: contract}}, nil
 }
 
 // Chequebook is an auto generated Go binding around an Ethereum contract.
 type Chequebook struct {
 	ChequebookCaller     // Read-only binding to the contract
 	ChequebookTransactor // Write-only binding to the contract
+	ChequebookFilterer   // Log filterer for contract events
 }
 
 // ChequebookCaller is an auto generated read-only Go binding around an Ethereum contract.
@@ -48,6 +51,11 @@ type ChequebookTransactor struct {
 	contract *bind.BoundContract // Generic contract wrapper for the low level calls
 }
 
+// ChequebookFilterer is an auto generated log filtering Go binding around an Ethereum contract events.
+type ChequebookFilterer struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
 // ChequebookSession is an auto generated Go binding around an Ethereum contract,
 // with pre-set call and transact options.
 type ChequebookSession struct {
@@ -87,16 +95,16 @@ type ChequebookTransactorRaw struct {
 
 // NewChequebook creates a new instance of Chequebook, bound to a specific deployed contract.
 func NewChequebook(address common.Address, backend bind.ContractBackend) (*Chequebook, error) {
-	contract, err := bindChequebook(address, backend, backend)
+	contract, err := bindChequebook(address, backend, backend, backend)
 	if err != nil {
 		return nil, err
 	}
-	return &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}}, nil
+	return &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}, ChequebookFilterer: ChequebookFilterer{contract: contract}}, nil
 }
 
 // NewChequebookCaller creates a new read-only instance of Chequebook, bound to a specific deployed contract.
 func NewChequebookCaller(address common.Address, caller bind.ContractCaller) (*ChequebookCaller, error) {
-	contract, err := bindChequebook(address, caller, nil)
+	contract, err := bindChequebook(address, caller, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -105,20 +113,29 @@ func NewChequebookCaller(address common.Address, caller bind.ContractCaller) (*C
 
 // NewChequebookTransactor creates a new write-only instance of Chequebook, bound to a specific deployed contract.
 func NewChequebookTransactor(address common.Address, transactor bind.ContractTransactor) (*ChequebookTransactor, error) {
-	contract, err := bindChequebook(address, nil, transactor)
+	contract, err := bindChequebook(address, nil, transactor, nil)
 	if err != nil {
 		return nil, err
 	}
 	return &ChequebookTransactor{contract: contract}, nil
 }
 
+// NewChequebookFilterer creates a new log filterer instance of Chequebook, bound to a specific deployed contract.
+func NewChequebookFilterer(address common.Address, filterer bind.ContractFilterer) (*ChequebookFilterer, error) {
+	contract, err := bindChequebook(address, nil, nil, filterer)
+	if err != nil {
+		return nil, err
+	}
+	return &ChequebookFilterer{contract: contract}, nil
+}
+
 // bindChequebook binds a generic wrapper to an already deployed contract.
-func bindChequebook(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) {
+func bindChequebook(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
 	parsed, err := abi.JSON(strings.NewReader(ChequebookABI))
 	if err != nil {
 		return nil, err
 	}
-	return bind.NewBoundContract(address, parsed, caller, transactor), nil
+	return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
 }
 
 // Call invokes the (constant) contract method with params as input values and
@@ -226,3 +243,125 @@ func (_Chequebook *ChequebookSession) Kill() (*types.Transaction, error) {
 func (_Chequebook *ChequebookTransactorSession) Kill() (*types.Transaction, error) {
 	return _Chequebook.Contract.Kill(&_Chequebook.TransactOpts)
 }
+
+// ChequebookOverdraftIterator is returned from FilterOverdraft and is used to iterate over the raw logs and unpacked data for Overdraft events raised by the Chequebook contract.
+type ChequebookOverdraftIterator struct {
+	Event *ChequebookOverdraft // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *ChequebookOverdraftIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(ChequebookOverdraft)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(ChequebookOverdraft)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error retruned any retrieval or parsing error occurred during filtering.
+func (it *ChequebookOverdraftIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *ChequebookOverdraftIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// ChequebookOverdraft represents a Overdraft event raised by the Chequebook contract.
+type ChequebookOverdraft struct {
+	Deadbeat common.Address
+	Raw      types.Log // Blockchain specific contextual infos
+}
+
+// FilterOverdraft is a free log retrieval operation binding the contract event 0x2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f978.
+//
+// Solidity: event Overdraft(deadbeat address)
+func (_Chequebook *ChequebookFilterer) FilterOverdraft(opts *bind.FilterOpts) (*ChequebookOverdraftIterator, error) {
+
+	logs, sub, err := _Chequebook.contract.FilterLogs(opts, "Overdraft")
+	if err != nil {
+		return nil, err
+	}
+	return &ChequebookOverdraftIterator{contract: _Chequebook.contract, event: "Overdraft", logs: logs, sub: sub}, nil
+}
+
+// WatchOverdraft is a free log subscription operation binding the contract event 0x2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f978.
+//
+// Solidity: event Overdraft(deadbeat address)
+func (_Chequebook *ChequebookFilterer) WatchOverdraft(opts *bind.WatchOpts, sink chan<- *ChequebookOverdraft) (event.Subscription, error) {
+
+	logs, sub, err := _Chequebook.contract.WatchLogs(opts, "Overdraft")
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(ChequebookOverdraft)
+				if err := _Chequebook.contract.UnpackLog(event, "Overdraft", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}

contracts/chequebook/contract/code.go

@@ -2,4 +2,4 @@ package contract
 
 // ContractDeployedCode is used to detect suicides. This constant needs to be
 // updated when the contract code is changed.
-const ContractDeployedCode = "0x6060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a7230582014e927522ca5cd8f68529ac4d3b9cdf36d40e09d8a33b70008248d1abebf79680029"
+const ContractDeployedCode = "0x6060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a72305820533e856fc37e3d64d1706bcc7dfb6b1d490c8d566ea498d9d01ec08965a896ca0029"

contracts/ens/contract/ens.go

@@ -6,17 +6,19 @@ package contract
 import (
 	"strings"
 
+	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
 )
 
 // ENSABI is the input ABI used to generate the binding from.
 const ENSABI = "[{\"constant\":true,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"resolver\",\"outputs\":[{\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"owner\",\"outputs\":[{\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"},{\"name\":\"label\",\"type\":\"bytes32\"},{\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"setSubnodeOwner\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"},{\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"setTTL\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"ttl\",\"outputs\":[{\"name\":\"\",\"type\":\"uint64\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"},{\"name\":\"resolver\",\"type\":\"address\"}],\"name\":\"setResolver\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"node\",\"type\":\"bytes32\"},{\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"setOwner\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":true,\"name\":\"label\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"NewOwner\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"Transfer\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"resolver\",\"type\":\"address\"}],\"name\":\"NewResolver\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"NewTTL\",\"type\":\"event\"}]"
 
 // ENSBin is the compiled bytecode used for deploying new contracts.
-const ENSBin = `0x6060604052341561000f57600080fd5b60008080526020527fad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb58054600160a060020a033316600160a060020a0319909116179055610503806100626000396000f3006060604052600436106100825763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416630178b8bf811461008757806302571be3146100b957806306ab5923146100cf57806314ab9038146100f657806316a25cbd146101195780631896f70a1461014c5780635b0fc9c31461016e575b600080fd5b341561009257600080fd5b61009d600435610190565b604051600160a060020a03909116815260200160405180910390f35b34156100c457600080fd5b61009d6004356101ae565b34156100da57600080fd5b6100f4600435602435600160a060020a03604435166101c9565b005b341561010157600080fd5b6100f460043567ffffffffffffffff6024351661028b565b341561012457600080fd5b61012f600435610357565b60405167ffffffffffffffff909116815260200160405180910390f35b341561015757600080fd5b6100f4600435600160a060020a036024351661038e565b341561017957600080fd5b6100f4600435600160a060020a0360243516610434565b600090815260208190526040902060010154600160a060020a031690565b600090815260208190526040902054600160a060020a031690565b600083815260208190526040812054849033600160a060020a039081169116146101f257600080fd5b8484604051918252602082015260409081019051908190039020915083857fce0457fe73731f824cc272376169235128c118b49d344817417c6d108d155e8285604051600160a060020a03909116815260200160405180910390a3506000908152602081905260409020805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a03929092169190911790555050565b600082815260208190526040902054829033600160a060020a039081169116146102b457600080fd5b827f1d4f9bbfc9cab89d66e1a1562f2233ccbf1308cb4f63de2ead5787adddb8fa688360405167ffffffffffffffff909116815260200160405180910390a250600091825260208290526040909120600101805467ffffffffffffffff90921674010000000000000000000000000000000000000000027fffffffff0000000000000000ffffffffffffffffffffffffffffffffffffffff909216919091179055565b60009081526020819052604090206001015474010000000000000000000000000000000000000000900467ffffffffffffffff1690565b600082815260208190526040902054829033600160a060020a039081169116146103b757600080fd5b827f335721b01866dc23fbee8b6b2c7b1e14d6f05c28cd35a2c934239f94095602a083604051600160a060020a03909116815260200160405180910390a250600091825260208290526040909120600101805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a03909216919091179055565b600082815260208190526040902054829033600160a060020a0390811691161461045d57600080fd5b827fd4735d920b0f87494915f556dd9b54c8f309026070caea5c737245152564d26683604051600160a060020a03909116815260200160405180910390a250600091825260208290526040909120805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a039092169190911790555600a165627a7a7230582087c335a130f7bd19015451f7e1dc0e44cdeb5b64393f51a105ee00160711fcff0029`
+const ENSBin = `0x6060604052341561000f57600080fd5b60008080526020527fad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb58054600160a060020a033316600160a060020a0319909116179055610503806100626000396000f3006060604052600436106100825763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416630178b8bf811461008757806302571be3146100b957806306ab5923146100cf57806314ab9038146100f657806316a25cbd146101195780631896f70a1461014c5780635b0fc9c31461016e575b600080fd5b341561009257600080fd5b61009d600435610190565b604051600160a060020a03909116815260200160405180910390f35b34156100c457600080fd5b61009d6004356101ae565b34156100da57600080fd5b6100f4600435602435600160a060020a03604435166101c9565b005b341561010157600080fd5b6100f460043567ffffffffffffffff6024351661028b565b341561012457600080fd5b61012f600435610357565b60405167ffffffffffffffff909116815260200160405180910390f35b341561015757600080fd5b6100f4600435600160a060020a036024351661038e565b341561017957600080fd5b6100f4600435600160a060020a0360243516610434565b600090815260208190526040902060010154600160a060020a031690565b600090815260208190526040902054600160a060020a031690565b600083815260208190526040812054849033600160a060020a039081169116146101f257600080fd5b8484604051918252602082015260409081019051908190039020915083857fce0457fe73731f824cc272376169235128c118b49d344817417c6d108d155e8285604051600160a060020a03909116815260200160405180910390a3506000908152602081905260409020805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a03929092169190911790555050565b600082815260208190526040902054829033600160a060020a039081169116146102b457600080fd5b827f1d4f9bbfc9cab89d66e1a1562f2233ccbf1308cb4f63de2ead5787adddb8fa688360405167ffffffffffffffff909116815260200160405180910390a250600091825260208290526040909120600101805467ffffffffffffffff90921674010000000000000000000000000000000000000000027fffffffff0000000000000000ffffffffffffffffffffffffffffffffffffffff909216919091179055565b60009081526020819052604090206001015474010000000000000000000000000000000000000000900467ffffffffffffffff1690565b600082815260208190526040902054829033600160a060020a039081169116146103b757600080fd5b827f335721b01866dc23fbee8b6b2c7b1e14d6f05c28cd35a2c934239f94095602a083604051600160a060020a03909116815260200160405180910390a250600091825260208290526040909120600101805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a03909216919091179055565b600082815260208190526040902054829033600160a060020a0390811691161461045d57600080fd5b827fd4735d920b0f87494915f556dd9b54c8f309026070caea5c737245152564d26683604051600160a060020a03909116815260200160405180910390a250600091825260208290526040909120805473ffffffffffffffffffffffffffffffffffffffff1916600160a060020a039092169190911790555600a165627a7a72305820f4c798d4c84c9912f389f64631e85e8d16c3e6644f8c2e1579936015c7d5f6660029`
 
 // DeployENS deploys a new Ethereum contract, binding an instance of ENS to it.
 func DeployENS(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *ENS, error) {
@@ -28,13 +30,14 @@ func DeployENS(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Ad
 	if err != nil {
 		return common.Address{}, nil, nil, err
 	}
-	return address, tx, &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}}, nil
+	return address, tx, &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}, ENSFilterer: ENSFilterer{contract: contract}}, nil
 }
 
 // ENS is an auto generated Go binding around an Ethereum contract.
 type ENS struct {
 	ENSCaller     // Read-only binding to the contract
 	ENSTransactor // Write-only binding to the contract
+	ENSFilterer   // Log filterer for contract events
 }
 
 // ENSCaller is an auto generated read-only Go binding around an Ethereum contract.
@@ -47,6 +50,11 @@ type ENSTransactor struct {
 	contract *bind.BoundContract // Generic contract wrapper for the low level calls
 }
 
+// ENSFilterer is an auto generated log filtering Go binding around an Ethereum contract events.
+type ENSFilterer struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
 // ENSSession is an auto generated Go binding around an Ethereum contract,
 // with pre-set call and transact options.
 type ENSSession struct {
@@ -86,16 +94,16 @@ type ENSTransactorRaw struct {
 
 // NewENS creates a new instance of ENS, bound to a specific deployed contract.
 func NewENS(address common.Address, backend bind.ContractBackend) (*ENS, error) {
-	contract, err := bindENS(address, backend, backend)
+	contract, err := bindENS(address, backend, backend, backend)
 	if err != nil {
 		return nil, err
 	}
-	return &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}}, nil
+	return &ENS{ENSCaller: ENSCaller{contract: contract}, ENSTransactor: ENSTransactor{contract: contract}, ENSFilterer: ENSFilterer{contract: contract}}, nil
 }
 
 // NewENSCaller creates a new read-only instance of ENS, bound to a specific deployed contract.
 func NewENSCaller(address common.Address, caller bind.ContractCaller) (*ENSCaller, error) {
-	contract, err := bindENS(address, caller, nil)
+	contract, err := bindENS(address, caller, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -104,20 +112,29 @@ func NewENSCaller(address common.Address, caller bind.ContractCaller) (*ENSCalle
 
 // NewENSTransactor creates a new write-only instance of ENS, bound to a specific deployed contract.
 func NewENSTransactor(address common.Address, transactor bind.ContractTransactor) (*ENSTransactor, error) {
-	contract, err := bindENS(address, nil, transactor)
+	contract, err := bindENS(address, nil, transactor, nil)
 	if err != nil {
 		return nil, err
 	}
 	return &ENSTransactor{contract: contract}, nil
 }
 
+// NewENSFilterer creates a new log filterer instance of ENS, bound to a specific deployed contract.
+func NewENSFilterer(address common.Address, filterer bind.ContractFilterer) (*ENSFilterer, error) {
+	contract, err := bindENS(address, nil, nil, filterer)
+	if err != nil {
+		return nil, err
+	}
+	return &ENSFilterer{contract: contract}, nil
+}
+
 // bindENS binds a generic wrapper to an already deployed contract.
-func bindENS(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) {
+func bindENS(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
 	parsed, err := abi.JSON(strings.NewReader(ENSABI))
 	if err != nil {
 		return nil, err
 	}
-	return bind.NewBoundContract(address, parsed, caller, transactor), nil
+	return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
 }
 
 // Call invokes the (constant) contract method with params as input values and
@@ -319,3 +336,544 @@ func (_ENS *ENSSession) SetTTL(node [32]byte, ttl uint64) (*types.Transaction, e
 func (_ENS *ENSTransactorSession) SetTTL(node [32]byte, ttl uint64) (*types.Transaction, error) {
 	return _ENS.Contract.SetTTL(&_ENS.TransactOpts, node, ttl)
 }
+
+// ENSNewOwnerIterator is returned from FilterNewOwner and is used to iterate over the raw logs and unpacked data for NewOwner events raised by the ENS contract.
+type ENSNewOwnerIterator struct {
+	Event *ENSNewOwner // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *ENSNewOwnerIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(ENSNewOwner)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(ENSNewOwner)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error retruned any retrieval or parsing error occurred during filtering.
+func (it *ENSNewOwnerIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *ENSNewOwnerIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// ENSNewOwner represents a NewOwner event raised by the ENS contract.
+type ENSNewOwner struct {
+	Node  [32]byte
+	Label [32]byte
+	Owner common.Address
+	Raw   types.Log // Blockchain specific contextual infos
+}
+
+// FilterNewOwner is a free log retrieval operation binding the contract event 0xce0457fe73731f824cc272376169235128c118b49d344817417c6d108d155e82.
+//
+// Solidity: event NewOwner(node indexed bytes32, label indexed bytes32, owner address)
+func (_ENS *ENSFilterer) FilterNewOwner(opts *bind.FilterOpts, node [][32]byte, label [][32]byte) (*ENSNewOwnerIterator, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+	var labelRule []interface{}
+	for _, labelItem := range label {
+		labelRule = append(labelRule, labelItem)
+	}
+
+	logs, sub, err := _ENS.contract.FilterLogs(opts, "NewOwner", nodeRule, labelRule)
+	if err != nil {
+		return nil, err
+	}
+	return &ENSNewOwnerIterator{contract: _ENS.contract, event: "NewOwner", logs: logs, sub: sub}, nil
+}
+
+// WatchNewOwner is a free log subscription operation binding the contract event 0xce0457fe73731f824cc272376169235128c118b49d344817417c6d108d155e82.
+//
+// Solidity: event NewOwner(node indexed bytes32, label indexed bytes32, owner address)
+func (_ENS *ENSFilterer) WatchNewOwner(opts *bind.WatchOpts, sink chan<- *ENSNewOwner, node [][32]byte, label [][32]byte) (event.Subscription, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+	var labelRule []interface{}
+	for _, labelItem := range label {
+		labelRule = append(labelRule, labelItem)
+	}
+
+	logs, sub, err := _ENS.contract.WatchLogs(opts, "NewOwner", nodeRule, labelRule)
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(ENSNewOwner)
+				if err := _ENS.contract.UnpackLog(event, "NewOwner", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// ENSNewResolverIterator is returned from FilterNewResolver and is used to iterate over the raw logs and unpacked data for NewResolver events raised by the ENS contract.
+type ENSNewResolverIterator struct {
+	Event *ENSNewResolver // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *ENSNewResolverIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(ENSNewResolver)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(ENSNewResolver)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error retruned any retrieval or parsing error occurred during filtering.
+func (it *ENSNewResolverIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *ENSNewResolverIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// ENSNewResolver represents a NewResolver event raised by the ENS contract.
+type ENSNewResolver struct {
+	Node     [32]byte
+	Resolver common.Address
+	Raw      types.Log // Blockchain specific contextual infos
+}
+
+// FilterNewResolver is a free log retrieval operation binding the contract event 0x335721b01866dc23fbee8b6b2c7b1e14d6f05c28cd35a2c934239f94095602a0.
+//
+// Solidity: event NewResolver(node indexed bytes32, resolver address)
+func (_ENS *ENSFilterer) FilterNewResolver(opts *bind.FilterOpts, node [][32]byte) (*ENSNewResolverIterator, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.FilterLogs(opts, "NewResolver", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return &ENSNewResolverIterator{contract: _ENS.contract, event: "NewResolver", logs: logs, sub: sub}, nil
+}
+
+// WatchNewResolver is a free log subscription operation binding the contract event 0x335721b01866dc23fbee8b6b2c7b1e14d6f05c28cd35a2c934239f94095602a0.
+//
+// Solidity: event NewResolver(node indexed bytes32, resolver address)
+func (_ENS *ENSFilterer) WatchNewResolver(opts *bind.WatchOpts, sink chan<- *ENSNewResolver, node [][32]byte) (event.Subscription, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.WatchLogs(opts, "NewResolver", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(ENSNewResolver)
+				if err := _ENS.contract.UnpackLog(event, "NewResolver", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// ENSNewTTLIterator is returned from FilterNewTTL and is used to iterate over the raw logs and unpacked data for NewTTL events raised by the ENS contract.
+type ENSNewTTLIterator struct {
+	Event *ENSNewTTL // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *ENSNewTTLIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(ENSNewTTL)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(ENSNewTTL)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error retruned any retrieval or parsing error occurred during filtering.
+func (it *ENSNewTTLIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *ENSNewTTLIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// ENSNewTTL represents a NewTTL event raised by the ENS contract.
+type ENSNewTTL struct {
+	Node [32]byte
+	Ttl  uint64
+	Raw  types.Log // Blockchain specific contextual infos
+}
+
+// FilterNewTTL is a free log retrieval operation binding the contract event 0x1d4f9bbfc9cab89d66e1a1562f2233ccbf1308cb4f63de2ead5787adddb8fa68.
+//
+// Solidity: event NewTTL(node indexed bytes32, ttl uint64)
+func (_ENS *ENSFilterer) FilterNewTTL(opts *bind.FilterOpts, node [][32]byte) (*ENSNewTTLIterator, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.FilterLogs(opts, "NewTTL", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return &ENSNewTTLIterator{contract: _ENS.contract, event: "NewTTL", logs: logs, sub: sub}, nil
+}
+
+// WatchNewTTL is a free log subscription operation binding the contract event 0x1d4f9bbfc9cab89d66e1a1562f2233ccbf1308cb4f63de2ead5787adddb8fa68.
+//
+// Solidity: event NewTTL(node indexed bytes32, ttl uint64)
+func (_ENS *ENSFilterer) WatchNewTTL(opts *bind.WatchOpts, sink chan<- *ENSNewTTL, node [][32]byte) (event.Subscription, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.WatchLogs(opts, "NewTTL", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(ENSNewTTL)
+				if err := _ENS.contract.UnpackLog(event, "NewTTL", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// ENSTransferIterator is returned from FilterTransfer and is used to iterate over the raw logs and unpacked data for Transfer events raised by the ENS contract.
+type ENSTransferIterator struct {
+	Event *ENSTransfer // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *ENSTransferIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(ENSTransfer)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(ENSTransfer)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error retruned any retrieval or parsing error occurred during filtering.
+func (it *ENSTransferIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *ENSTransferIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// ENSTransfer represents a Transfer event raised by the ENS contract.
+type ENSTransfer struct {
+	Node  [32]byte
+	Owner common.Address
+	Raw   types.Log // Blockchain specific contextual infos
+}
+
+// FilterTransfer is a free log retrieval operation binding the contract event 0xd4735d920b0f87494915f556dd9b54c8f309026070caea5c737245152564d266.
+//
+// Solidity: event Transfer(node indexed bytes32, owner address)
+func (_ENS *ENSFilterer) FilterTransfer(opts *bind.FilterOpts, node [][32]byte) (*ENSTransferIterator, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.FilterLogs(opts, "Transfer", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return &ENSTransferIterator{contract: _ENS.contract, event: "Transfer", logs: logs, sub: sub}, nil
+}
+
+// WatchTransfer is a free log subscription operation binding the contract event 0xd4735d920b0f87494915f556dd9b54c8f309026070caea5c737245152564d266.
+//
+// Solidity: event Transfer(node indexed bytes32, owner address)
+func (_ENS *ENSFilterer) WatchTransfer(opts *bind.WatchOpts, sink chan<- *ENSTransfer, node [][32]byte) (event.Subscription, error) {
+
+	var nodeRule []interface{}
+	for _, nodeItem := range node {
+		nodeRule = append(nodeRule, nodeItem)
+	}
+
+	logs, sub, err := _ENS.contract.WatchLogs(opts, "Transfer", nodeRule)
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(ENSTransfer)
+				if err := _ENS.contract.UnpackLog(event, "Transfer", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}

contracts/ens/contract/fifsregistrar.go

@@ -16,7 +16,7 @@ import (
 const FIFSRegistrarABI = "[{\"constant\":false,\"inputs\":[{\"name\":\"subnode\",\"type\":\"bytes32\"},{\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"register\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"ensAddr\",\"type\":\"address\"},{\"name\":\"node\",\"type\":\"bytes32\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"}]"
 
 // FIFSRegistrarBin is the compiled bytecode used for deploying new contracts.
-const FIFSRegistrarBin = `0x6060604052341561000f57600080fd5b604051604080610224833981016040528080519190602001805160008054600160a060020a03909516600160a060020a03199095169490941790935550506001556101c58061005f6000396000f3006060604052600436106100275763ffffffff60e060020a600035041663d22057a9811461002c575b600080fd5b341561003757600080fd5b61004e600435600160a060020a0360243516610050565b005b816000806001548360405191825260208201526040908101905190819003902060008054919350600160a060020a03909116906302571be39084906040516020015260405160e060020a63ffffffff84160281526004810191909152602401602060405180830381600087803b15156100c857600080fd5b6102c65a03f115156100d957600080fd5b5050506040518051915050600160a060020a0381161580159061010e575033600160a060020a031681600160a060020a031614155b1561011857600080fd5b600054600154600160a060020a03909116906306ab592390878760405160e060020a63ffffffff861602815260048101939093526024830191909152600160a060020a03166044820152606401600060405180830381600087803b151561017e57600080fd5b6102c65a03f1151561018f57600080fd5b50505050505050505600a165627a7a723058209b0c0f4ed76e4fe49a71d4b838ab3d00d6bad29021172db7ced9f36abcafbf510029`
+const FIFSRegistrarBin = `0x6060604052341561000f57600080fd5b604051604080610224833981016040528080519190602001805160008054600160a060020a03909516600160a060020a03199095169490941790935550506001556101c58061005f6000396000f3006060604052600436106100275763ffffffff60e060020a600035041663d22057a9811461002c575b600080fd5b341561003757600080fd5b61004e600435600160a060020a0360243516610050565b005b816000806001548360405191825260208201526040908101905190819003902060008054919350600160a060020a03909116906302571be39084906040516020015260405160e060020a63ffffffff84160281526004810191909152602401602060405180830381600087803b15156100c857600080fd5b6102c65a03f115156100d957600080fd5b5050506040518051915050600160a060020a0381161580159061010e575033600160a060020a031681600160a060020a031614155b1561011857600080fd5b600054600154600160a060020a03909116906306ab592390878760405160e060020a63ffffffff861602815260048101939093526024830191909152600160a060020a03166044820152606401600060405180830381600087803b151561017e57600080fd5b6102c65a03f1151561018f57600080fd5b50505050505050505600a165627a7a723058206fb963cb168d5e3a51af12cd6bb23e324dbd32dd4954f43653ba27e66b68ea650029`
 
 // DeployFIFSRegistrar deploys a new Ethereum contract, binding an instance of FIFSRegistrar to it.
 func DeployFIFSRegistrar(auth *bind.TransactOpts, backend bind.ContractBackend, ensAddr common.Address, node [32]byte) (common.Address, *types.Transaction, *FIFSRegistrar, error) {
@@ -28,13 +28,14 @@ func DeployFIFSRegistrar(auth *bind.TransactOpts, backend bind.ContractBackend,
 	if err != nil {
 		return common.Address{}, nil, nil, err
 	}
-	return address, tx, &FIFSRegistrar{FIFSRegistrarCaller: FIFSRegistrarCaller{contract: contract}, FIFSRegistrarTransactor: FIFSRegistrarTransactor{contract: contract}}, nil
+	return address, tx, &FIFSRegistrar{FIFSRegistrarCaller: FIFSRegistrarCaller{contract: contract}, FIFSRegistrarTransactor: FIFSRegistrarTransactor{contract: contract}, FIFSRegistrarFilterer: FIFSRegistrarFilterer{contract: contract}}, nil
 }
 
 // FIFSRegistrar is an auto generated Go binding around an Ethereum contract.
 type FIFSRegistrar struct {
 	FIFSRegistrarCaller     // Read-only binding to the contract
 	FIFSRegistrarTransactor // Write-only binding to the contract
+	FIFSRegistrarFilterer   // Log filterer for contract events
 }
 
 // FIFSRegistrarCaller is an auto generated read-only Go binding around an Ethereum contract.
@@ -47,6 +48,11 @@ type FIFSRegistrarTransactor struct {
 	contract *bind.BoundContract // Generic contract wrapper for the low level calls
 }
 
+// FIFSRegistrarFilterer is an auto generated log filtering Go binding around an Ethereum contract events.
+type FIFSRegistrarFilterer struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
 // FIFSRegistrarSession is an auto generated Go binding around an Ethereum contract,
 // with pre-set call and transact options.
 type FIFSRegistrarSession struct {
@@ -86,16 +92,16 @@ type FIFSRegistrarTransactorRaw struct {
 
 // NewFIFSRegistrar creates a new instance of FIFSRegistrar, bound to a specific deployed contract.
 func NewFIFSRegistrar(address common.Address, backend bind.ContractBackend) (*FIFSRegistrar, error) {
-	contract, err := bindFIFSRegistrar(address, backend, backend)
+	contract, err := bindFIFSRegistrar(address, backend, backend, backend)
 	if err != nil {
 		return nil, err
 	}
-	return &FIFSRegistrar{FIFSRegistrarCaller: FIFSRegistrarCaller{contract: contract}, FIFSRegistrarTransactor: FIFSRegistrarTransactor{contract: contract}}, nil
+	return &FIFSRegistrar{FIFSRegistrarCaller: FIFSRegistrarCaller{contract: contract}, FIFSRegistrarTransactor: FIFSRegistrarTransactor{contract: contract}, FIFSRegistrarFilterer: FIFSRegistrarFilterer{contract: contract}}, nil
 }
 
 // NewFIFSRegistrarCaller creates a new read-only instance of FIFSRegistrar, bound to a specific deployed contract.
 func NewFIFSRegistrarCaller(address common.Address, caller bind.ContractCaller) (*FIFSRegistrarCaller, error) {
-	contract, err := bindFIFSRegistrar(address, caller, nil)
+	contract, err := bindFIFSRegistrar(address, caller, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -104,20 +110,29 @@ func NewFIFSRegistrarCaller(address common.Address, caller bind.ContractCaller)
 
 // NewFIFSRegistrarTransactor creates a new write-only instance of FIFSRegistrar, bound to a specific deployed contract.
 func NewFIFSRegistrarTransactor(address common.Address, transactor bind.ContractTransactor) (*FIFSRegistrarTransactor, error) {
-	contract, err := bindFIFSRegistrar(address, nil, transactor)
+	contract, err := bindFIFSRegistrar(address, nil, transactor, nil)
 	if err != nil {
 		return nil, err
 	}
 	return &FIFSRegistrarTransactor{contract: contract}, nil
 }
 
+// NewFIFSRegistrarFilterer creates a new log filterer instance of FIFSRegistrar, bound to a specific deployed contract.
+func NewFIFSRegistrarFilterer(address common.Address, filterer bind.ContractFilterer) (*FIFSRegistrarFilterer, error) {
+	contract, err := bindFIFSRegistrar(address, nil, nil, filterer)
+	if err != nil {
+		return nil, err
+	}
+	return &FIFSRegistrarFilterer{contract: contract}, nil
+}
+
 // bindFIFSRegistrar binds a generic wrapper to an already deployed contract.
-func bindFIFSRegistrar(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) {
+func bindFIFSRegistrar(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
 	parsed, err := abi.JSON(strings.NewReader(FIFSRegistrarABI))
 	if err != nil {
 		return nil, err
 	}
-	return bind.NewBoundContract(address, parsed, caller, transactor), nil
+	return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
 }
 
 // Call invokes the (constant) contract method with params as input values and

contracts/release/contract.sol

@@ -1,251 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-pragma solidity ^0.4.18;
-
-// ReleaseOracle is an Ethereum contract to store the current and previous
-// versions of the go-ethereum implementation. Its goal is to allow Geth to
-// check for new releases automatically without the need to consult a central
-// repository.
-//
-// The contract takes a vote based approach on both assigning authorised signers
-// as well as signing off on new Geth releases.
-//
-// Note, when a signer is demoted, the currently pending release is auto-nuked.
-// The reason is to prevent suprises where a demotion actually tilts the votes
-// in favor of one voter party and pushing out a new release as a consequence of
-// a simple demotion.
-contract ReleaseOracle {
-  // Votes is an internal data structure to count votes on a specific proposal
-  struct Votes {
-    address[] pass; // List of signers voting to pass a proposal
-    address[] fail; // List of signers voting to fail a proposal
-  }
-
-  // Version is the version details of a particular Geth release
-  struct Version {
-    uint32  major;  // Major version component of the release
-    uint32  minor;  // Minor version component of the release
-    uint32  patch;  // Patch version component of the release
-    bytes20 commit; // Git SHA1 commit hash of the release
-
-    uint64  time;  // Timestamp of the release approval
-    Votes   votes; // Votes that passed this release
-  }
-
-  // Oracle authorization details
-  mapping(address => bool) authorised; // Set of accounts allowed to vote on updating the contract
-  address[]                voters;     // List of addresses currently accepted as signers
-
-  // Various proposals being voted on
-  mapping(address => Votes) authProps; // Currently running user authorization proposals
-  address[]                 authPend;  // List of addresses being voted on (map indexes)
-
-  Version   verProp;  // Currently proposed release being voted on
-  Version[] releases; // All the positively voted releases
-
-  // isSigner is a modifier to authorize contract transactions.
-  modifier isSigner() {
-    if (authorised[msg.sender]) {
-      _;
-    }
-  }
-
-  // Constructor to assign the initial set of signers.
-  function ReleaseOracle(address[] signers) {
-    // If no signers were specified, assign the creator as the sole signer
-    if (signers.length == 0) {
-      authorised[msg.sender] = true;
-      voters.push(msg.sender);
-      return;
-    }
-    // Otherwise assign the individual signers one by one
-    for (uint i = 0; i < signers.length; i++) {
-      authorised[signers[i]] = true;
-      voters.push(signers[i]);
-    }
-  }
-
-  // signers is an accessor method to retrieve all the signers (public accessor
-  // generates an indexed one, not a retrieve-all version).
-  function signers() constant returns(address[]) {
-    return voters;
-  }
-
-  // authProposals retrieves the list of addresses that authorization proposals
-  // are currently being voted on.
-  function authProposals() constant returns(address[]) {
-    return authPend;
-  }
-
-  // authVotes retrieves the current authorization votes for a particular user
-  // to promote him into the list of signers, or demote him from there.
-  function authVotes(address user) constant returns(address[] promote, address[] demote) {
-    return (authProps[user].pass, authProps[user].fail);
-  }
-
-  // currentVersion retrieves the semantic version, commit hash and release time
-  // of the currently votec active release.
-  function currentVersion() constant returns (uint32 major, uint32 minor, uint32 patch, bytes20 commit, uint time) {
-    if (releases.length == 0) {
-      return (0, 0, 0, 0, 0);
-    }
-    var release = releases[releases.length - 1];
-
-    return (release.major, release.minor, release.patch, release.commit, release.time);
-  }
-
-  // proposedVersion retrieves the semantic version, commit hash and the current
-  // votes for the next proposed release.
-  function proposedVersion() constant returns (uint32 major, uint32 minor, uint32 patch, bytes20 commit, address[] pass, address[] fail) {
-    return (verProp.major, verProp.minor, verProp.patch, verProp.commit, verProp.votes.pass, verProp.votes.fail);
-  }
-
-  // promote pitches in on a voting campaign to promote a new user to a signer
-  // position.
-  function promote(address user) {
-    updateSigner(user, true);
-  }
-
-  // demote pitches in on a voting campaign to demote an authorised user from
-  // its signer position.
-  function demote(address user) {
-    updateSigner(user, false);
-  }
-
-  // release votes for a particular version to be included as the next release.
-  function release(uint32 major, uint32 minor, uint32 patch, bytes20 commit) {
-    updateRelease(major, minor, patch, commit, true);
-  }
-
-  // nuke votes for the currently proposed version to not be included as the next
-  // release. Nuking doesn't require a specific version number for simplicity.
-  function nuke() {
-    updateRelease(0, 0, 0, 0, false);
-  }
-
-  // updateSigner marks a vote for changing the status of an Ethereum user, either
-  // for or against the user being an authorised signer.
-  function updateSigner(address user, bool authorize) internal isSigner {
-    // Gather the current votes and ensure we don't double vote
-    Votes votes = authProps[user];
-    for (uint i = 0; i < votes.pass.length; i++) {
-      if (votes.pass[i] == msg.sender) {
-        return;
-      }
-    }
-    for (i = 0; i < votes.fail.length; i++) {
-      if (votes.fail[i] == msg.sender) {
-        return;
-      }
-    }
-    // If no authorization proposal is open, add the user to the index for later lookups
-    if (votes.pass.length == 0 && votes.fail.length == 0) {
-      authPend.push(user);
-    }
-    // Cast the vote and return if the proposal cannot be resolved yet
-    if (authorize) {
-      votes.pass.push(msg.sender);
-      if (votes.pass.length <= voters.length / 2) {
-        return;
-      }
-    } else {
-      votes.fail.push(msg.sender);
-      if (votes.fail.length <= voters.length / 2) {
-        return;
-      }
-    }
-    // Proposal resolved in our favor, execute whatever we voted on
-    if (authorize && !authorised[user]) {
-      authorised[user] = true;
-      voters.push(user);
-    } else if (!authorize && authorised[user]) {
-      authorised[user] = false;
-
-      for (i = 0; i < voters.length; i++) {
-        if (voters[i] == user) {
-          voters[i] = voters[voters.length - 1];
-          voters.length--;
-
-          delete verProp; // Nuke any version proposal (no surprise releases!)
-          break;
-        }
-      }
-    }
-    // Finally delete the resolved proposal, index and garbage collect
-    delete authProps[user];
-
-    for (i = 0; i < authPend.length; i++) {
-      if (authPend[i] == user) {
-        authPend[i] = authPend[authPend.length - 1];
-        authPend.length--;
-        break;
-      }
-    }
-  }
-
-  // updateRelease votes for a particular version to be included as the next release,
-  // or for the currently proposed release to be nuked out.
-  function updateRelease(uint32 major, uint32 minor, uint32 patch, bytes20 commit, bool release) internal isSigner {
-    // Skip nuke votes if no proposal is pending
-    if (!release && verProp.votes.pass.length == 0) {
-      return;
-    }
-    // Mark a new release if no proposal is pending
-    if (verProp.votes.pass.length == 0) {
-      verProp.major  = major;
-      verProp.minor  = minor;
-      verProp.patch  = patch;
-      verProp.commit = commit;
-    }
-    // Make sure positive votes match the current proposal
-    if (release && (verProp.major != major || verProp.minor != minor || verProp.patch != patch || verProp.commit != commit)) {
-      return;
-    }
-    // Gather the current votes and ensure we don't double vote
-    Votes votes = verProp.votes;
-    for (uint i = 0; i < votes.pass.length; i++) {
-      if (votes.pass[i] == msg.sender) {
-        return;
-      }
-    }
-    for (i = 0; i < votes.fail.length; i++) {
-      if (votes.fail[i] == msg.sender) {
-        return;
-      }
-    }
-    // Cast the vote and return if the proposal cannot be resolved yet
-    if (release) {
-      votes.pass.push(msg.sender);
-      if (votes.pass.length <= voters.length / 2) {
-        return;
-      }
-    } else {
-      votes.fail.push(msg.sender);
-      if (votes.fail.length <= voters.length / 2) {
-        return;
-      }
-    }
-    // Proposal resolved in our favor, execute whatever we voted on
-    if (release) {
-      verProp.time = uint64(now);
-      releases.push(verProp);
-      delete verProp;
-    } else {
-      delete verProp;
-    }
-  }
-}

contracts/release/contract_test.go

@@ -1,374 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package release
-
-import (
-	"crypto/ecdsa"
-	"math/big"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-// setupReleaseTest creates a blockchain simulator and deploys a version oracle
-// contract for testing.
-func setupReleaseTest(t *testing.T, prefund ...*ecdsa.PrivateKey) (*ecdsa.PrivateKey, *ReleaseOracle, *backends.SimulatedBackend) {
-	// Generate a new random account and a funded simulator
-	key, _ := crypto.GenerateKey()
-	auth := bind.NewKeyedTransactor(key)
-
-	alloc := core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}
-	for _, key := range prefund {
-		alloc[crypto.PubkeyToAddress(key.PublicKey)] = core.GenesisAccount{Balance: big.NewInt(10000000000)}
-	}
-	sim := backends.NewSimulatedBackend(alloc)
-
-	// Deploy a version oracle contract, commit and return
-	_, _, oracle, err := DeployReleaseOracle(auth, sim, []common.Address{auth.From})
-	if err != nil {
-		t.Fatalf("Failed to deploy version contract: %v", err)
-	}
-	sim.Commit()
-
-	return key, oracle, sim
-}
-
-// Tests that the version contract can be deployed and the creator is assigned
-// the sole authorized signer.
-func TestContractCreation(t *testing.T) {
-	key, oracle, _ := setupReleaseTest(t)
-
-	owner := crypto.PubkeyToAddress(key.PublicKey)
-	signers, err := oracle.Signers(nil)
-	if err != nil {
-		t.Fatalf("Failed to retrieve list of signers: %v", err)
-	}
-	if len(signers) != 1 || signers[0] != owner {
-		t.Fatalf("Initial signer mismatch: have %v, want %v", signers, owner)
-	}
-}
-
-// Tests that subsequent signers can be promoted, each requiring half plus one
-// votes for it to pass through.
-func TestSignerPromotion(t *testing.T) {
-	// Prefund a few accounts to authorize with and create the oracle
-	keys := make([]*ecdsa.PrivateKey, 5)
-	for i := 0; i < len(keys); i++ {
-		keys[i], _ = crypto.GenerateKey()
-	}
-	key, oracle, sim := setupReleaseTest(t, keys...)
-
-	// Gradually promote the keys, until all are authorized
-	keys = append([]*ecdsa.PrivateKey{key}, keys...)
-	for i := 1; i < len(keys); i++ {
-		// Check that no votes are accepted from the not yet authorized user
-		if _, err := oracle.Promote(bind.NewKeyedTransactor(keys[i]), common.Address{}); err != nil {
-			t.Fatalf("Iter #%d: failed invalid promotion attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		pend, err := oracle.AuthProposals(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve active proposals: %v", i, err)
-		}
-		if len(pend) != 0 {
-			t.Fatalf("Iter #%d: proposal count mismatch: have %d, want 0", i, len(pend))
-		}
-		// Promote with half - 1 voters and check that the user's not yet authorized
-		for j := 0; j < i/2; j++ {
-			if _, err = oracle.Promote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid promotion attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-
-		signers, err := oracle.Signers(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve list of signers: %v", i, err)
-		}
-		if len(signers) != i {
-			t.Fatalf("Iter #%d: signer count mismatch: have %v, want %v", i, len(signers), i)
-		}
-		// Promote with the last one needed to pass the promotion
-		if _, err = oracle.Promote(bind.NewKeyedTransactor(keys[i/2]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-			t.Fatalf("Iter #%d: failed valid promotion completion attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		signers, err = oracle.Signers(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve list of signers: %v", i, err)
-		}
-		if len(signers) != i+1 {
-			t.Fatalf("Iter #%d: signer count mismatch: have %v, want %v", i, len(signers), i+1)
-		}
-	}
-}
-
-// Tests that subsequent signers can be demoted, each requiring half plus one
-// votes for it to pass through.
-func TestSignerDemotion(t *testing.T) {
-	// Prefund a few accounts to authorize with and create the oracle
-	keys := make([]*ecdsa.PrivateKey, 5)
-	for i := 0; i < len(keys); i++ {
-		keys[i], _ = crypto.GenerateKey()
-	}
-	key, oracle, sim := setupReleaseTest(t, keys...)
-
-	// Authorize all the keys as valid signers and verify cardinality
-	keys = append([]*ecdsa.PrivateKey{key}, keys...)
-	for i := 1; i < len(keys); i++ {
-		for j := 0; j <= i/2; j++ {
-			if _, err := oracle.Promote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid promotion attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-	}
-	signers, err := oracle.Signers(nil)
-	if err != nil {
-		t.Fatalf("Failed to retrieve list of signers: %v", err)
-	}
-	if len(signers) != len(keys) {
-		t.Fatalf("Signer count mismatch: have %v, want %v", len(signers), len(keys))
-	}
-	// Gradually demote users until we run out of signers
-	for i := len(keys) - 1; i >= 0; i-- {
-		// Demote with half - 1 voters and check that the user's not yet dropped
-		for j := 0; j < (i+1)/2; j++ {
-			if _, err = oracle.Demote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid demotion attempt: %v", len(keys)-i, err)
-			}
-		}
-		sim.Commit()
-
-		signers, err := oracle.Signers(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve list of signers: %v", len(keys)-i, err)
-		}
-		if len(signers) != i+1 {
-			t.Fatalf("Iter #%d: signer count mismatch: have %v, want %v", len(keys)-i, len(signers), i+1)
-		}
-		// Demote with the last one needed to pass the demotion
-		if _, err = oracle.Demote(bind.NewKeyedTransactor(keys[(i+1)/2]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-			t.Fatalf("Iter #%d: failed valid demotion completion attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		signers, err = oracle.Signers(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve list of signers: %v", len(keys)-i, err)
-		}
-		if len(signers) != i {
-			t.Fatalf("Iter #%d: signer count mismatch: have %v, want %v", len(keys)-i, len(signers), i)
-		}
-		// Check that no votes are accepted from the already demoted users
-		if _, err = oracle.Promote(bind.NewKeyedTransactor(keys[i]), common.Address{}); err != nil {
-			t.Fatalf("Iter #%d: failed invalid promotion attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		pend, err := oracle.AuthProposals(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve active proposals: %v", i, err)
-		}
-		if len(pend) != 0 {
-			t.Fatalf("Iter #%d: proposal count mismatch: have %d, want 0", i, len(pend))
-		}
-	}
-}
-
-// Tests that new versions can be released, honouring both voting rights as well
-// as the minimum required vote count.
-func TestVersionRelease(t *testing.T) {
-	// Prefund a few accounts to authorize with and create the oracle
-	keys := make([]*ecdsa.PrivateKey, 5)
-	for i := 0; i < len(keys); i++ {
-		keys[i], _ = crypto.GenerateKey()
-	}
-	key, oracle, sim := setupReleaseTest(t, keys...)
-
-	// Track the "current release"
-	var (
-		verMajor  = uint32(0)
-		verMinor  = uint32(0)
-		verPatch  = uint32(0)
-		verCommit = [20]byte{}
-	)
-	// Gradually push releases, always requiring more signers than previously
-	keys = append([]*ecdsa.PrivateKey{key}, keys...)
-	for i := 1; i < len(keys); i++ {
-		// Check that no votes are accepted from the not yet authorized user
-		if _, err := oracle.Release(bind.NewKeyedTransactor(keys[i]), 0, 0, 0, [20]byte{0}); err != nil {
-			t.Fatalf("Iter #%d: failed invalid release attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		prop, err := oracle.ProposedVersion(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve active proposal: %v", i, err)
-		}
-		if len(prop.Pass) != 0 {
-			t.Fatalf("Iter #%d: proposal vote count mismatch: have %d, want 0", i, len(prop.Pass))
-		}
-		// Authorize the user to make releases
-		for j := 0; j <= i/2; j++ {
-			if _, err = oracle.Promote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid promotion attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-
-		// Propose release with half voters and check that the release does not yet go through
-		for j := 0; j < (i+1)/2; j++ {
-			if _, err = oracle.Release(bind.NewKeyedTransactor(keys[j]), uint32(i), uint32(i+1), uint32(i+2), [20]byte{byte(i + 3)}); err != nil {
-				t.Fatalf("Iter #%d: failed valid release attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-
-		ver, err := oracle.CurrentVersion(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve current version: %v", i, err)
-		}
-		if ver.Major != verMajor || ver.Minor != verMinor || ver.Patch != verPatch || ver.Commit != verCommit {
-			t.Fatalf("Iter #%d: version mismatch: have %d.%d.%d-%x, want %d.%d.%d-%x", i, ver.Major, ver.Minor, ver.Patch, ver.Commit, verMajor, verMinor, verPatch, verCommit)
-		}
-
-		// Pass the release and check that it became the next version
-		verMajor, verMinor, verPatch, verCommit = uint32(i), uint32(i+1), uint32(i+2), [20]byte{byte(i + 3)}
-		if _, err = oracle.Release(bind.NewKeyedTransactor(keys[(i+1)/2]), uint32(i), uint32(i+1), uint32(i+2), [20]byte{byte(i + 3)}); err != nil {
-			t.Fatalf("Iter #%d: failed valid release completion attempt: %v", i, err)
-		}
-		sim.Commit()
-
-		ver, err = oracle.CurrentVersion(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve current version: %v", i, err)
-		}
-		if ver.Major != verMajor || ver.Minor != verMinor || ver.Patch != verPatch || ver.Commit != verCommit {
-			t.Fatalf("Iter #%d: version mismatch: have %d.%d.%d-%x, want %d.%d.%d-%x", i, ver.Major, ver.Minor, ver.Patch, ver.Commit, verMajor, verMinor, verPatch, verCommit)
-		}
-	}
-}
-
-// Tests that proposed versions can be nuked out of existence.
-func TestVersionNuking(t *testing.T) {
-	// Prefund a few accounts to authorize with and create the oracle
-	keys := make([]*ecdsa.PrivateKey, 9)
-	for i := 0; i < len(keys); i++ {
-		keys[i], _ = crypto.GenerateKey()
-	}
-	key, oracle, sim := setupReleaseTest(t, keys...)
-
-	// Authorize all the keys as valid signers
-	keys = append([]*ecdsa.PrivateKey{key}, keys...)
-	for i := 1; i < len(keys); i++ {
-		for j := 0; j <= i/2; j++ {
-			if _, err := oracle.Promote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid promotion attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-	}
-	// Propose releases with more and more keys, always retaining enough users to nuke the proposals
-	for i := 1; i < (len(keys)+1)/2; i++ {
-		// Propose release with an initial set of signers
-		for j := 0; j < i; j++ {
-			if _, err := oracle.Release(bind.NewKeyedTransactor(keys[j]), uint32(i), uint32(i+1), uint32(i+2), [20]byte{byte(i + 3)}); err != nil {
-				t.Fatalf("Iter #%d: failed valid proposal attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-
-		prop, err := oracle.ProposedVersion(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve active proposal: %v", i, err)
-		}
-		if len(prop.Pass) != i {
-			t.Fatalf("Iter #%d: proposal vote count mismatch: have %d, want %d", i, len(prop.Pass), i)
-		}
-		// Nuke the release with half+1 voters
-		for j := i; j <= i+(len(keys)+1)/2; j++ {
-			if _, err := oracle.Nuke(bind.NewKeyedTransactor(keys[j])); err != nil {
-				t.Fatalf("Iter #%d: failed valid nuke attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-
-		prop, err = oracle.ProposedVersion(nil)
-		if err != nil {
-			t.Fatalf("Iter #%d: failed to retrieve active proposal: %v", i, err)
-		}
-		if len(prop.Pass) != 0 || len(prop.Fail) != 0 {
-			t.Fatalf("Iter #%d: proposal vote count mismatch: have %d/%d pass/fail, want 0/0", i, len(prop.Pass), len(prop.Fail))
-		}
-	}
-}
-
-// Tests that demoting a signer will auto-nuke the currently pending release.
-func TestVersionAutoNuke(t *testing.T) {
-	// Prefund a few accounts to authorize with and create the oracle
-	keys := make([]*ecdsa.PrivateKey, 5)
-	for i := 0; i < len(keys); i++ {
-		keys[i], _ = crypto.GenerateKey()
-	}
-	key, oracle, sim := setupReleaseTest(t, keys...)
-
-	// Authorize all the keys as valid signers
-	keys = append([]*ecdsa.PrivateKey{key}, keys...)
-	for i := 1; i < len(keys); i++ {
-		for j := 0; j <= i/2; j++ {
-			if _, err := oracle.Promote(bind.NewKeyedTransactor(keys[j]), crypto.PubkeyToAddress(keys[i].PublicKey)); err != nil {
-				t.Fatalf("Iter #%d: failed valid promotion attempt: %v", i, err)
-			}
-		}
-		sim.Commit()
-	}
-	// Make a release proposal and check it's existence
-	if _, err := oracle.Release(bind.NewKeyedTransactor(keys[0]), 1, 2, 3, [20]byte{4}); err != nil {
-		t.Fatalf("Failed valid proposal attempt: %v", err)
-	}
-	sim.Commit()
-
-	prop, err := oracle.ProposedVersion(nil)
-	if err != nil {
-		t.Fatalf("Failed to retrieve active proposal: %v", err)
-	}
-	if len(prop.Pass) != 1 {
-		t.Fatalf("Proposal vote count mismatch: have %d, want 1", len(prop.Pass))
-	}
-	// Demote a signer and check release proposal deletion
-	for i := 0; i <= len(keys)/2; i++ {
-		if _, err := oracle.Demote(bind.NewKeyedTransactor(keys[i]), crypto.PubkeyToAddress(keys[len(keys)-1].PublicKey)); err != nil {
-			t.Fatalf("Iter #%d: failed valid demotion attempt: %v", i, err)
-		}
-	}
-	sim.Commit()
-
-	prop, err = oracle.ProposedVersion(nil)
-	if err != nil {
-		t.Fatalf("Failed to retrieve active proposal: %v", err)
-	}
-	if len(prop.Pass) != 0 {
-		t.Fatalf("Proposal vote count mismatch: have %d, want 0", len(prop.Pass))
-	}
-}

contracts/release/release.go

@@ -1,164 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package release contains the node service that tracks client releases.
-package release
-
-//go:generate abigen --sol ./contract.sol --pkg release --out ./contract.go
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/internal/ethapi"
-	"github.com/ethereum/go-ethereum/les"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/rpc"
-)
-
-// Interval to check for new releases
-const releaseRecheckInterval = time.Hour
-
-// Config contains the configurations of the release service.
-type Config struct {
-	Oracle common.Address // Ethereum address of the release oracle
-	Major  uint32         // Major version component of the release
-	Minor  uint32         // Minor version component of the release
-	Patch  uint32         // Patch version component of the release
-	Commit [20]byte       // Git SHA1 commit hash of the release
-}
-
-// ReleaseService is a node service that periodically checks the blockchain for
-// newly released versions of the client being run and issues a warning to the
-// user about it.
-type ReleaseService struct {
-	config Config          // Current version to check releases against
-	oracle *ReleaseOracle  // Native binding to the release oracle contract
-	quit   chan chan error // Quit channel to terminate the version checker
-}
-
-// NewReleaseService creates a new service to periodically check for new client
-// releases and notify the user of such.
-func NewReleaseService(ctx *node.ServiceContext, config Config) (node.Service, error) {
-	// Retrieve the Ethereum service dependency to access the blockchain
-	var apiBackend ethapi.Backend
-	var ethereum *eth.Ethereum
-	if err := ctx.Service(&ethereum); err == nil {
-		apiBackend = ethereum.ApiBackend
-	} else {
-		var ethereum *les.LightEthereum
-		if err := ctx.Service(&ethereum); err == nil {
-			apiBackend = ethereum.ApiBackend
-		} else {
-			return nil, err
-		}
-	}
-	// Construct the release service
-	contract, err := NewReleaseOracle(config.Oracle, eth.NewContractBackend(apiBackend))
-	if err != nil {
-		return nil, err
-	}
-	return &ReleaseService{
-		config: config,
-		oracle: contract,
-		quit:   make(chan chan error),
-	}, nil
-}
-
-// Protocols returns an empty list of P2P protocols as the release service does
-// not have a networking component.
-func (r *ReleaseService) Protocols() []p2p.Protocol { return nil }
-
-// APIs returns an empty list of RPC descriptors as the release service does not
-// expose any functioanlity to the outside world.
-func (r *ReleaseService) APIs() []rpc.API { return nil }
-
-// Start spawns the periodic version checker goroutine
-func (r *ReleaseService) Start(server *p2p.Server) error {
-	go r.checker()
-	return nil
-}
-
-// Stop terminates all goroutines belonging to the service, blocking until they
-// are all terminated.
-func (r *ReleaseService) Stop() error {
-	errc := make(chan error)
-	r.quit <- errc
-	return <-errc
-}
-
-// checker runs indefinitely in the background, periodically checking for new
-// client releases.
-func (r *ReleaseService) checker() {
-	// Set up the timers to periodically check for releases
-	timer := time.NewTimer(0) // Immediately fire a version check
-	defer timer.Stop()
-
-	for {
-		select {
-		case <-timer.C:
-			// Rechedule the timer before continuing
-			timer.Reset(releaseRecheckInterval)
-			r.checkVersion()
-		case errc := <-r.quit:
-			errc <- nil
-			return
-		}
-	}
-}
-
-func (r *ReleaseService) checkVersion() {
-	// Retrieve the current version, and handle missing contracts gracefully
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	opts := &bind.CallOpts{Context: ctx}
-	defer cancel()
-
-	version, err := r.oracle.CurrentVersion(opts)
-	if err != nil {
-		if err == bind.ErrNoCode {
-			log.Debug("Release oracle not found", "contract", r.config.Oracle)
-		} else if err != les.ErrNoPeers {
-			log.Error("Failed to retrieve current release", "err", err)
-		}
-		return
-	}
-	// Version was successfully retrieved, notify if newer than ours
-	if version.Major > r.config.Major ||
-		(version.Major == r.config.Major && version.Minor > r.config.Minor) ||
-		(version.Major == r.config.Major && version.Minor == r.config.Minor && version.Patch > r.config.Patch) {
-
-		warning := fmt.Sprintf("Client v%d.%d.%d-%x seems older than the latest upstream release v%d.%d.%d-%x",
-			r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4], version.Major, version.Minor, version.Patch, version.Commit[:4])
-		howtofix := fmt.Sprintf("Please check https://github.com/ethereum/go-ethereum/releases for new releases")
-		separator := strings.Repeat("-", len(warning))
-
-		log.Warn(separator)
-		log.Warn(warning)
-		log.Warn(howtofix)
-		log.Warn(separator)
-	} else {
-		log.Debug("Client seems up to date with upstream",
-			"local", fmt.Sprintf("v%d.%d.%d-%x", r.config.Major, r.config.Minor, r.config.Patch, r.config.Commit[:4]),
-			"upstream", fmt.Sprintf("v%d.%d.%d-%x", version.Major, version.Minor, version.Patch, version.Commit[:4]))
-	}
-}

core/bench_test.go

@@ -173,7 +173,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 
 	// Time the insertion of the new chain.
 	// State and blocks are stored in the same DB.
-	chainman, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer chainman.Stop()
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
-		chain, err := NewBlockChain(db, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+		chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
 		if err != nil {
 			b.Fatalf("error creating chain: %v", err)
 		}

core/block_validator.go

@@ -50,11 +50,14 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin
 // validated at this point.
 func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	// Check whether the block's known, and if not, that it's linkable
-	if v.bc.HasBlockAndState(block.Hash()) {
+	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
 		return ErrKnownBlock
 	}
-	if !v.bc.HasBlockAndState(block.ParentHash()) {
+	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
-		return consensus.ErrUnknownAncestor
+		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+			return consensus.ErrUnknownAncestor
+		}
+		return consensus.ErrPrunedAncestor
 	}
 	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()

core/block_validator_test.go

@@ -42,7 +42,7 @@ func TestHeaderVerification(t *testing.T) {
 		headers[i] = block.Header()
 	}
 	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
-	chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
 	defer chain.Stop()
 
 	for i := 0; i < len(blocks); i++ {
@@ -106,11 +106,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) {
 		var results <-chan error
 
 		if valid {
-			chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
 			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
 			chain.Stop()
 		} else {
-			chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{})
+			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{})
 			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
 			chain.Stop()
 		}
@@ -173,7 +173,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) {
 	defer runtime.GOMAXPROCS(old)
 
 	// Start the verifications and immediately abort
-	chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{})
+	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{})
 	defer chain.Stop()
 
 	abort, results := chain.engine.VerifyHeaders(chain, headers, seals)

core/blockchain.go

@@ -42,6 +42,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/hashicorp/golang-lru"
+	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
 )
 
 var (
@@ -56,11 +57,20 @@ const (
 	maxFutureBlocks     = 256
 	maxTimeFutureBlocks = 30
 	badBlockLimit       = 10
+	triesInMemory       = 128
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
 	BlockChainVersion = 3
 )
 
+// CacheConfig contains the configuration values for the trie caching/pruning
+// that's resident in a blockchain.
+type CacheConfig struct {
+	Disabled      bool          // Whether to disable trie write caching (archive node)
+	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
+	TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
+}
+
 // BlockChain represents the canonical chain given a database with a genesis
 // block. The Blockchain manages chain imports, reverts, chain reorganisations.
 //
@@ -76,10 +86,14 @@ const (
 // included in the canonical one where as GetBlockByNumber always represents the
 // canonical chain.
 type BlockChain struct {
-	config *params.ChainConfig // chain & network configuration
+	chainConfig *params.ChainConfig // Chain & network configuration
+	cacheConfig *CacheConfig        // Cache configuration for pruning
+
+	db     ethdb.Database // Low level persistent database to store final content in
+	triegc *prque.Prque   // Priority queue mapping block numbers to tries to gc
+	gcproc time.Duration  // Accumulates canonical block processing for trie dumping
 
 	hc            *HeaderChain
-	chainDb       ethdb.Database
 	rmLogsFeed    event.Feed
 	chainFeed     event.Feed
 	chainSideFeed event.Feed
@@ -119,7 +133,13 @@ type BlockChain struct {
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialises the default Ethereum Validator and
 // Processor.
-func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
+func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
+	if cacheConfig == nil {
+		cacheConfig = &CacheConfig{
+			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieTimeLimit: 5 * time.Minute,
+		}
+	}
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
@@ -127,9 +147,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
 	badBlocks, _ := lru.New(badBlockLimit)
 
 	bc := &BlockChain{
-		config:       config,
+		chainConfig:  chainConfig,
-		chainDb:      chainDb,
+		cacheConfig:  cacheConfig,
-		stateCache:   state.NewDatabase(chainDb),
+		db:           db,
+		triegc:       prque.New(),
+		stateCache:   state.NewDatabase(db),
 		quit:         make(chan struct{}),
 		bodyCache:    bodyCache,
 		bodyRLPCache: bodyRLPCache,
@@ -139,11 +161,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
 		vmConfig:     vmConfig,
 		badBlocks:    badBlocks,
 	}
-	bc.SetValidator(NewBlockValidator(config, bc, engine))
+	bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
-	bc.SetProcessor(NewStateProcessor(config, bc, engine))
+	bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
 
 	var err error
-	bc.hc, err = NewHeaderChain(chainDb, config, engine, bc.getProcInterrupt)
+	bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt)
 	if err != nil {
 		return nil, err
 	}
@@ -180,7 +202,7 @@ func (bc *BlockChain) getProcInterrupt() bool {
 // assumes that the chain manager mutex is held.
 func (bc *BlockChain) loadLastState() error {
 	// Restore the last known head block
-	head := GetHeadBlockHash(bc.chainDb)
+	head := GetHeadBlockHash(bc.db)
 	if head == (common.Hash{}) {
 		// Corrupt or empty database, init from scratch
 		log.Warn("Empty database, resetting chain")
@@ -196,15 +218,17 @@ func (bc *BlockChain) loadLastState() error {
 	// Make sure the state associated with the block is available
 	if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
 		// Dangling block without a state associated, init from scratch
-		log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
+		log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
-		return bc.Reset()
+		if err := bc.repair(&currentBlock); err != nil {
+			return err
+		}
 	}
 	// Everything seems to be fine, set as the head block
 	bc.currentBlock = currentBlock
 
 	// Restore the last known head header
 	currentHeader := bc.currentBlock.Header()
-	if head := GetHeadHeaderHash(bc.chainDb); head != (common.Hash{}) {
+	if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) {
 		if header := bc.GetHeaderByHash(head); header != nil {
 			currentHeader = header
 		}
@@ -213,7 +237,7 @@ func (bc *BlockChain) loadLastState() error {
 
 	// Restore the last known head fast block
 	bc.currentFastBlock = bc.currentBlock
-	if head := GetHeadFastBlockHash(bc.chainDb); head != (common.Hash{}) {
+	if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) {
 		if block := bc.GetBlockByHash(head); block != nil {
 			bc.currentFastBlock = block
 		}
@@ -243,7 +267,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 
 	// Rewind the header chain, deleting all block bodies until then
 	delFn := func(hash common.Hash, num uint64) {
-		DeleteBody(bc.chainDb, hash, num)
+		DeleteBody(bc.db, hash, num)
 	}
 	bc.hc.SetHead(head, delFn)
 	currentHeader := bc.hc.CurrentHeader()
@@ -275,10 +299,10 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	if bc.currentFastBlock == nil {
 		bc.currentFastBlock = bc.genesisBlock
 	}
-	if err := WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()); err != nil {
+	if err := WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head full block", "err", err)
 	}
-	if err := WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()); err != nil {
+	if err := WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head fast block", "err", err)
 	}
 	return bc.loadLastState()
@@ -292,7 +316,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x…]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), bc.chainDb, 0); err != nil {
+	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil {
 		return err
 	}
 	// If all checks out, manually set the head block
@@ -312,14 +336,6 @@ func (bc *BlockChain) GasLimit() uint64 {
 	return bc.currentBlock.GasLimit()
 }
 
-// LastBlockHash return the hash of the HEAD block.
-func (bc *BlockChain) LastBlockHash() common.Hash {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentBlock.Hash()
-}
-
 // CurrentBlock retrieves the current head block of the canonical chain. The
 // block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentBlock() *types.Block {
@@ -338,15 +354,6 @@ func (bc *BlockChain) CurrentFastBlock() *types.Block {
 	return bc.currentFastBlock
 }
 
-// Status returns status information about the current chain such as the HEAD Td,
-// the HEAD hash and the hash of the genesis block.
-func (bc *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()), bc.currentBlock.Hash(), bc.genesisBlock.Hash()
-}
-
 // SetProcessor sets the processor required for making state modifications.
 func (bc *BlockChain) SetProcessor(processor Processor) {
 	bc.procmu.Lock()
@@ -404,7 +411,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil {
 		log.Crit("Failed to write genesis block TD", "err", err)
 	}
-	if err := WriteBlock(bc.chainDb, genesis); err != nil {
+	if err := WriteBlock(bc.db, genesis); err != nil {
 		log.Crit("Failed to write genesis block", "err", err)
 	}
 	bc.genesisBlock = genesis
@@ -417,6 +424,24 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	return nil
 }
 
+// repair tries to repair the current blockchain by rolling back the current block
+// until one with associated state is found. This is needed to fix incomplete db
+// writes caused either by crashes/power outages, or simply non-committed tries.
+//
+// This method only rolls back the current block. The current header and current
+// fast block are left intact.
+func (bc *BlockChain) repair(head **types.Block) error {
+	for {
+		// Abort if we've rewound to a head block that does have associated state
+		if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
+			log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash())
+			return nil
+		}
+		// Otherwise rewind one block and recheck state availability there
+		(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+	}
+}
+
 // Export writes the active chain to the given writer.
 func (bc *BlockChain) Export(w io.Writer) error {
 	return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64())
@@ -454,22 +479,22 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
 // Note, this function assumes that the `mu` mutex is held!
 func (bc *BlockChain) insert(block *types.Block) {
 	// If the block is on a side chain or an unknown one, force other heads onto it too
-	updateHeads := GetCanonicalHash(bc.chainDb, block.NumberU64()) != block.Hash()
+	updateHeads := GetCanonicalHash(bc.db, block.NumberU64()) != block.Hash()
 
 	// Add the block to the canonical chain number scheme and mark as the head
-	if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil {
+	if err := WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()); err != nil {
 		log.Crit("Failed to insert block number", "err", err)
 	}
-	if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil {
+	if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil {
 		log.Crit("Failed to insert head block hash", "err", err)
 	}
 	bc.currentBlock = block
 
-	// If the block is better than out head or is on a different chain, force update heads
+	// If the block is better than our head or is on a different chain, force update heads
 	if updateHeads {
 		bc.hc.SetCurrentHeader(block.Header())
 
-		if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil {
+		if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil {
 			log.Crit("Failed to insert head fast block hash", "err", err)
 		}
 		bc.currentFastBlock = block
@@ -489,7 +514,7 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body {
 		body := cached.(*types.Body)
 		return body
 	}
-	body := GetBody(bc.chainDb, hash, bc.hc.GetBlockNumber(hash))
+	body := GetBody(bc.db, hash, bc.hc.GetBlockNumber(hash))
 	if body == nil {
 		return nil
 	}
@@ -505,7 +530,7 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue {
 	if cached, ok := bc.bodyRLPCache.Get(hash); ok {
 		return cached.(rlp.RawValue)
 	}
-	body := GetBodyRLP(bc.chainDb, hash, bc.hc.GetBlockNumber(hash))
+	body := GetBodyRLP(bc.db, hash, bc.hc.GetBlockNumber(hash))
 	if len(body) == 0 {
 		return nil
 	}
@@ -519,21 +544,25 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	if bc.blockCache.Contains(hash) {
 		return true
 	}
-	ok, _ := bc.chainDb.Has(blockBodyKey(hash, number))
+	ok, _ := bc.db.Has(blockBodyKey(hash, number))
 	return ok
 }
 
+// HasState checks if state trie is fully present in the database or not.
+func (bc *BlockChain) HasState(hash common.Hash) bool {
+	_, err := bc.stateCache.OpenTrie(hash)
+	return err == nil
+}
+
 // HasBlockAndState checks if a block and associated state trie is fully present
 // in the database or not, caching it if present.
-func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool {
+func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
 	// Check first that the block itself is known
-	block := bc.GetBlockByHash(hash)
+	block := bc.GetBlock(hash, number)
 	if block == nil {
 		return false
 	}
-	// Ensure the associated state is also present
+	return bc.HasState(block.Root())
-	_, err := bc.stateCache.OpenTrie(block.Root())
-	return err == nil
 }
 
 // GetBlock retrieves a block from the database by hash and number,
@@ -543,7 +572,7 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block {
 	if block, ok := bc.blockCache.Get(hash); ok {
 		return block.(*types.Block)
 	}
-	block := GetBlock(bc.chainDb, hash, number)
+	block := GetBlock(bc.db, hash, number)
 	if block == nil {
 		return nil
 	}
@@ -560,13 +589,18 @@ func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
 // GetBlockByNumber retrieves a block from the database by number, caching it
 // (associated with its hash) if found.
 func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block {
-	hash := GetCanonicalHash(bc.chainDb, number)
+	hash := GetCanonicalHash(bc.db, number)
 	if hash == (common.Hash{}) {
 		return nil
 	}
 	return bc.GetBlock(hash, number)
 }
 
+// GetReceiptsByHash retrieves the receipts for all transactions in a given block.
+func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+	return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash))
+}
+
 // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.
 // [deprecated by eth/62]
 func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) {
@@ -594,6 +628,12 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.
 	return uncles
 }
 
+// TrieNode retrieves a blob of data associated with a trie node (or code hash)
+// either from ephemeral in-memory cache, or from persistent storage.
+func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
+	return bc.stateCache.TrieDB().Node(hash)
+}
+
 // Stop stops the blockchain service. If any imports are currently in progress
 // it will abort them using the procInterrupt.
 func (bc *BlockChain) Stop() {
@@ -606,6 +646,33 @@ func (bc *BlockChain) Stop() {
 	atomic.StoreInt32(&bc.procInterrupt, 1)
 
 	bc.wg.Wait()
+
+	// Ensure the state of a recent block is also stored to disk before exiting.
+	// It is fine if this state does not exist (fast start/stop cycle), but it is
+	// advisable to leave an N block gap from the head so 1) a restart loads up
+	// the last N blocks as sync assistance to remote nodes; 2) a restart during
+	// a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from
+	// missing states are constantly tested.
+	//
+	// This may be tuned a bit on mainnet if its too annoying to reprocess the last
+	// N blocks.
+	if !bc.cacheConfig.Disabled {
+		triedb := bc.stateCache.TrieDB()
+		if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory {
+			recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1)
+
+			log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+			if err := triedb.Commit(recent.Root(), true); err != nil {
+				log.Error("Failed to commit recent state trie", "err", err)
+			}
+		}
+		for !bc.triegc.Empty() {
+			triedb.Dereference(bc.triegc.PopItem().(common.Hash), common.Hash{})
+		}
+		if size := triedb.Size(); size != 0 {
+			log.Error("Dangling trie nodes after full cleanup")
+		}
+	}
 	log.Info("Blockchain manager stopped")
 }
 
@@ -650,11 +717,11 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
 		}
 		if bc.currentFastBlock.Hash() == hash {
 			bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1)
-			WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash())
+			WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash())
 		}
 		if bc.currentBlock.Hash() == hash {
 			bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1)
-			WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash())
+			WriteHeadBlockHash(bc.db, bc.currentBlock.Hash())
 		}
 	}
 }
@@ -713,7 +780,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		stats = struct{ processed, ignored int32 }{}
 		start = time.Now()
 		bytes = 0
-		batch = bc.chainDb.NewBatch()
+		batch = bc.db.NewBatch()
 	)
 	for i, block := range blockChain {
 		receipts := receiptChain[i]
@@ -731,7 +798,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			continue
 		}
 		// Compute all the non-consensus fields of the receipts
-		SetReceiptsData(bc.config, block, receipts)
+		SetReceiptsData(bc.chainConfig, block, receipts)
 		// Write all the data out into the database
 		if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil {
 			return i, fmt.Errorf("failed to write block body: %v", err)
@@ -749,7 +816,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 				return 0, err
 			}
 			bytes += batch.ValueSize()
-			batch = bc.chainDb.NewBatch()
+			batch.Reset()
 		}
 	}
 	if batch.ValueSize() > 0 {
@@ -764,7 +831,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	head := blockChain[len(blockChain)-1]
 	if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case
 		if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 {
-			if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil {
+			if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil {
 				log.Crit("Failed to update head fast block hash", "err", err)
 			}
 			bc.currentFastBlock = head
@@ -775,15 +842,33 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	log.Info("Imported new block receipts",
 		"count", stats.processed,
 		"elapsed", common.PrettyDuration(time.Since(start)),
-		"bytes", bytes,
 		"number", head.Number(),
 		"hash", head.Hash(),
+		"size", common.StorageSize(bytes),
 		"ignored", stats.ignored)
 	return 0, nil
 }
 
-// WriteBlock writes the block to the chain.
+var lastWrite uint64
-func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) {
+
+// WriteBlockWithoutState writes only the block and its metadata to the database,
+// but does not write any state. This is used to construct competing side forks
+// up to the point where they exceed the canonical total difficulty.
+func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (err error) {
+	bc.wg.Add(1)
+	defer bc.wg.Done()
+
+	if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil {
+		return err
+	}
+	if err := WriteBlock(bc.db, block); err != nil {
+		return err
+	}
+	return nil
+}
+
+// WriteBlockWithState writes the block and all associated state to the database.
+func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) {
 	bc.wg.Add(1)
 	defer bc.wg.Done()
 
@@ -804,17 +889,73 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
 		return NonStatTy, err
 	}
 	// Write other block data using a batch.
-	batch := bc.chainDb.NewBatch()
+	batch := bc.db.NewBatch()
 	if err := WriteBlock(batch, block); err != nil {
 		return NonStatTy, err
 	}
-	if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil {
+	root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number()))
+	if err != nil {
 		return NonStatTy, err
 	}
+	triedb := bc.stateCache.TrieDB()
+
+	// If we're running an archive node, always flush
+	if bc.cacheConfig.Disabled {
+		if err := triedb.Commit(root, false); err != nil {
+			return NonStatTy, err
+		}
+	} else {
+		// Full but not archive node, do proper garbage collection
+		triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
+		bc.triegc.Push(root, -float32(block.NumberU64()))
+
+		if current := block.NumberU64(); current > triesInMemory {
+			// Find the next state trie we need to commit
+			header := bc.GetHeaderByNumber(current - triesInMemory)
+			chosen := header.Number.Uint64()
+
+			// Only write to disk if we exceeded our memory allowance *and* also have at
+			// least a given number of tries gapped.
+			var (
+				size  = triedb.Size()
+				limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+			)
+			if size > limit || bc.gcproc > bc.cacheConfig.TrieTimeLimit {
+				// If we're exceeding limits but haven't reached a large enough memory gap,
+				// warn the user that the system is becoming unstable.
+				if chosen < lastWrite+triesInMemory {
+					switch {
+					case size >= 2*limit:
+						log.Error("Trie memory critical, forcing to disk", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit:
+						log.Error("Trie timing critical, forcing to disk", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					case size > limit:
+						log.Warn("Trie memory at dangerous levels", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					case bc.gcproc > bc.cacheConfig.TrieTimeLimit:
+						log.Warn("Trie timing at dangerous levels", "time", bc.gcproc, "limit", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+					}
+				}
+				// If optimum or critical limits reached, write to disk
+				if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
+					triedb.Commit(header.Root, true)
+					lastWrite = chosen
+					bc.gcproc = 0
+				}
+			}
+			// Garbage collect anything below our required write retention
+			for !bc.triegc.Empty() {
+				root, number := bc.triegc.Pop()
+				if uint64(-number) > chosen {
+					bc.triegc.Push(root, number)
+					break
+				}
+				triedb.Dereference(root.(common.Hash), common.Hash{})
+			}
+		}
+	}
 	if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
 		return NonStatTy, err
 	}
-
 	// If the total difficulty is higher than our known, add it to the canonical chain
 	// Second clause in the if statement reduces the vulnerability to selfish mining.
 	// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
@@ -835,7 +976,7 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
 			return NonStatTy, err
 		}
 		// Write hash preimages
-		if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil {
+		if err := WritePreimages(bc.db, block.NumberU64(), state.Preimages()); err != nil {
 			return NonStatTy, err
 		}
 		status = CanonStatTy
@@ -927,31 +1068,60 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		if err == nil {
 			err = bc.Validator().ValidateBody(block)
 		}
-		if err != nil {
+		switch {
-			if err == ErrKnownBlock {
+		case err == ErrKnownBlock:
-				stats.ignored++
+			stats.ignored++
-				continue
+			continue
-			}
 
-			if err == consensus.ErrFutureBlock {
+		case err == consensus.ErrFutureBlock:
-				// Allow up to MaxFuture second in the future blocks. If this limit
+			// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
-				// is exceeded the chain is discarded and processed at a later time
+			// the chain is discarded and processed at a later time if given.
-				// if given.
+			max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
-				max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
+			if block.Time().Cmp(max) > 0 {
-				if block.Time().Cmp(max) > 0 {
+				return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
-					return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
+			}
+			bc.futureBlocks.Add(block.Hash(), block)
+			stats.queued++
+			continue
+
+		case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
+			bc.futureBlocks.Add(block.Hash(), block)
+			stats.queued++
+			continue
+
+		case err == consensus.ErrPrunedAncestor:
+			// Block competing with the canonical chain, store in the db, but don't process
+			// until the competitor TD goes above the canonical TD
+			localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
+			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
+			if localTd.Cmp(externTd) > 0 {
+				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
+					return i, events, coalescedLogs, err
 				}
-				bc.futureBlocks.Add(block.Hash(), block)
-				stats.queued++
 				continue
 			}
+			// Competitor chain beat canonical, gather all blocks from the common ancestor
+			var winner []*types.Block
 
-			if err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()) {
+			parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-				bc.futureBlocks.Add(block.Hash(), block)
+			for !bc.HasState(parent.Root()) {
-				stats.queued++
+				winner = append(winner, parent)
-				continue
+				parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
+			}
+			for j := 0; j < len(winner)/2; j++ {
+				winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
+			}
+			// Import all the pruned blocks to make the state available
+			bc.chainmu.Unlock()
+			_, evs, logs, err := bc.insertChain(winner)
+			bc.chainmu.Lock()
+			events, coalescedLogs = evs, logs
+
+			if err != nil {
+				return i, events, coalescedLogs, err
 			}
 
+		case err != nil:
 			bc.reportBlock(block, nil, err)
 			return i, events, coalescedLogs, err
 		}
@@ -979,8 +1149,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			bc.reportBlock(block, receipts, err)
 			return i, events, coalescedLogs, err
 		}
+		proctime := time.Since(bstart)
+
 		// Write the block to the chain and get the status.
-		status, err := bc.WriteBlockAndState(block, receipts, state)
+		status, err := bc.WriteBlockWithState(block, receipts, state)
 		if err != nil {
 			return i, events, coalescedLogs, err
 		}
@@ -994,6 +1166,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			events = append(events, ChainEvent{block, block.Hash(), logs})
 			lastCanon = block
 
+			// Only count canonical blocks for GC processing time
+			bc.gcproc += proctime
+
 		case SideStatTy:
 			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
 				common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
@@ -1003,10 +1178,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		}
 		stats.processed++
 		stats.usedGas += usedGas
-		stats.report(chain, i)
+		stats.report(chain, i, bc.stateCache.TrieDB().Size())
 	}
 	// Append a single chain head event if we've progressed the chain
-	if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() {
+	if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
 		events = append(events, ChainHeadEvent{lastCanon})
 	}
 	return 0, events, coalescedLogs, nil
@@ -1026,7 +1201,7 @@ const statsReportLimit = 8 * time.Second
 
 // report prints statistics if some number of blocks have been processed
 // or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int) {
+func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
 	// Fetch the timings for the batch
 	var (
 		now     = mclock.Now()
@@ -1041,7 +1216,7 @@ func (st *insertStats) report(chain []*types.Block, index int) {
 		context := []interface{}{
 			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
 			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
-			"number", end.Number(), "hash", end.Hash(),
+			"number", end.Number(), "hash", end.Hash(), "cache", cache,
 		}
 		if st.queued > 0 {
 			context = append(context, []interface{}{"queued", st.queued}...)
@@ -1077,7 +1252,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 		// These logs are later announced as deleted.
 		collectLogs = func(h common.Hash) {
 			// Coalesce logs and set 'Removed'.
-			receipts := GetBlockReceipts(bc.chainDb, h, bc.hc.GetBlockNumber(h))
+			receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h))
 			for _, receipt := range receipts {
 				for _, log := range receipt.Logs {
 					del := *log
@@ -1140,24 +1315,23 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 	} else {
 		log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash())
 	}
+	// Insert the new chain, taking care of the proper incremental order
 	var addedTxs types.Transactions
-	// insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly
+	for i := len(newChain) - 1; i >= 0; i-- {
-	for _, block := range newChain {
 		// insert the block in the canonical way, re-writing history
-		bc.insert(block)
+		bc.insert(newChain[i])
 		// write lookup entries for hash based transaction/receipt searches
-		if err := WriteTxLookupEntries(bc.chainDb, block); err != nil {
+		if err := WriteTxLookupEntries(bc.db, newChain[i]); err != nil {
 			return err
 		}
-		addedTxs = append(addedTxs, block.Transactions()...)
+		addedTxs = append(addedTxs, newChain[i].Transactions()...)
 	}
-
 	// calculate the difference between deleted and added transactions
 	diff := types.TxDifference(deletedTxs, addedTxs)
 	// When transactions get deleted from the database that means the
 	// receipts that were created in the fork must also be deleted
 	for _, tx := range diff {
-		DeleteTxLookupEntry(bc.chainDb, tx.Hash())
+		DeleteTxLookupEntry(bc.db, tx.Hash())
 	}
 	if len(deletedLogs) > 0 {
 		go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs})
@@ -1249,7 +1423,7 @@ Hash: 0x%x
 
 Error: %v
 ##############################
-`, bc.config, block.Number(), block.Hash(), receiptString, err))
+`, bc.chainConfig, block.Number(), block.Hash(), receiptString, err))
 }
 
 // InsertHeaderChain attempts to insert the given header chain in to the local
@@ -1356,7 +1530,7 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
 }
 
 // Config retrieves the blockchain's chain configuration.
-func (bc *BlockChain) Config() *params.ChainConfig { return bc.config }
+func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig }
 
 // Engine retrieves the blockchain's consensus engine.
 func (bc *BlockChain) Engine() consensus.Engine { return bc.engine }

core/blockchain_test.go

@@ -46,7 +46,7 @@ func newTestBlockChain(fake bool) *BlockChain {
 	if !fake {
 		engine = ethash.NewTester()
 	}
-	blockchain, err := NewBlockChain(db, gspec.Config, engine, vm.Config{})
+	blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
 	if err != nil {
 		panic(err)
 	}
@@ -148,9 +148,9 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 			return err
 		}
 		blockchain.mu.Lock()
-		WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash())))
+		WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash())))
-		WriteBlock(blockchain.chainDb, block)
+		WriteBlock(blockchain.db, block)
-		statedb.CommitTo(blockchain.chainDb, false)
+		statedb.Commit(false)
 		blockchain.mu.Unlock()
 	}
 	return nil
@@ -166,8 +166,8 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
 		}
 		// Manually insert the header into the database, but don't reorganise (allows subsequent testing)
 		blockchain.mu.Lock()
-		WriteTd(blockchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash)))
+		WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash)))
-		WriteHeader(blockchain.chainDb, header)
+		WriteHeader(blockchain.db, header)
 		blockchain.mu.Unlock()
 	}
 	return nil
@@ -186,9 +186,9 @@ func TestLastBlock(t *testing.T) {
 	bchain := newTestBlockChain(false)
 	defer bchain.Stop()
 
-	block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.chainDb, 0)[0]
+	block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0]
 	bchain.insert(block)
-	if block.Hash() != GetHeadBlockHash(bchain.chainDb) {
+	if block.Hash() != GetHeadBlockHash(bchain.db) {
 		t.Errorf("Write/Get HeadBlockHash failed")
 	}
 }
@@ -496,7 +496,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 	}
 
 	// Create a new BlockChain and check that it rolled back the state.
-	ncm, err := NewBlockChain(bc.chainDb, bc.config, ethash.NewFaker(), vm.Config{})
+	ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{})
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
@@ -609,7 +609,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Import the chain as an archive node for the comparison baseline
 	archiveDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
-	archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{})
+	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer archive.Stop()
 
 	if n, err := archive.InsertChain(blocks); err != nil {
@@ -618,7 +618,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Fast import the chain as a non-archive node to test
 	fastDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer fast.Stop()
 
 	headers := make([]*types.Header, len(blocks))
@@ -696,7 +696,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	archiveDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
 
-	archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{})
+	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
 	}
@@ -709,7 +709,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a non-archive node and ensure all pointers are updated
 	fastDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer fast.Stop()
 
 	headers := make([]*types.Header, len(blocks))
@@ -730,7 +730,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	lightDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(lightDb)
 
-	light, _ := NewBlockChain(lightDb, gspec.Config, ethash.NewFaker(), vm.Config{})
+	light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	if n, err := light.InsertHeaderChain(headers, 1); err != nil {
 		t.Fatalf("failed to insert header %d: %v", n, err)
 	}
@@ -799,7 +799,7 @@ func TestChainTxReorgs(t *testing.T) {
 		}
 	})
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		t.Fatalf("failed to insert original chain[%d]: %v", i, err)
 	}
@@ -870,7 +870,7 @@ func TestLogReorgs(t *testing.T) {
 		signer  = types.NewEIP155Signer(gspec.Config.ChainId)
 	)
 
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer blockchain.Stop()
 
 	rmLogsCh := make(chan RemovedLogsEvent)
@@ -917,7 +917,7 @@ func TestReorgSideEvent(t *testing.T) {
 		signer  = types.NewEIP155Signer(gspec.Config.ChainId)
 	)
 
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer blockchain.Stop()
 
 	chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {})
@@ -992,7 +992,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 	bc := newTestBlockChain(true)
 	defer bc.Stop()
 
-	chain, _ := GenerateChain(bc.config, bc.genesisBlock, ethash.NewFaker(), bc.chainDb, 10, func(i int, gen *BlockGen) {})
+	chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {})
 
 	var pend sync.WaitGroup
 	pend.Add(len(chain))
@@ -1003,14 +1003,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 
 			// try to retrieve a block by its canonical hash and see if the block data can be retrieved.
 			for {
-				ch := GetCanonicalHash(bc.chainDb, block.NumberU64())
+				ch := GetCanonicalHash(bc.db, block.NumberU64())
 				if ch == (common.Hash{}) {
 					continue // busy wait for canonical hash to be written
 				}
 				if ch != block.Hash() {
 					t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex())
 				}
-				fb := GetBlock(bc.chainDb, ch, block.NumberU64())
+				fb := GetBlock(bc.db, ch, block.NumberU64())
 				if fb == nil {
 					t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex())
 				}
@@ -1043,7 +1043,7 @@ func TestEIP155Transition(t *testing.T) {
 		genesis = gspec.MustCommit(db)
 	)
 
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer blockchain.Stop()
 
 	blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) {
@@ -1151,7 +1151,7 @@ func TestEIP161AccountRemoval(t *testing.T) {
 		}
 		genesis = gspec.MustCommit(db)
 	)
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer blockchain.Stop()
 
 	blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) {
@@ -1197,3 +1197,150 @@ func TestEIP161AccountRemoval(t *testing.T) {
 		t.Error("account should not exist")
 	}
 }
+
+// This is a regression test (i.e. as weird as it is, don't delete it ever), which
+// tests that under weird reorg conditions the blockchain and its internal header-
+// chain return the same latest block/header.
+//
+// https://github.com/ethereum/go-ethereum/pull/15941
+func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
+	// Generate a canonical chain to act as the main dataset
+	engine := ethash.NewFaker()
+
+	db, _ := ethdb.NewMemDatabase()
+	genesis := new(Genesis).MustCommit(db)
+	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
+
+	// Generate a bunch of fork blocks, each side forking from the canonical chain
+	forks := make([]*types.Block, len(blocks))
+	for i := 0; i < len(forks); i++ {
+		parent := genesis
+		if i > 0 {
+			parent = blocks[i-1]
+		}
+		fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) })
+		forks[i] = fork[0]
+	}
+	// Import the canonical and fork chain side by side, verifying the current block
+	// and current header consistency
+	diskdb, _ := ethdb.NewMemDatabase()
+	new(Genesis).MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	for i := 0; i < len(blocks); i++ {
+		if _, err := chain.InsertChain(blocks[i : i+1]); err != nil {
+			t.Fatalf("block %d: failed to insert into chain: %v", i, err)
+		}
+		if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() {
+			t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
+		}
+		if _, err := chain.InsertChain(forks[i : i+1]); err != nil {
+			t.Fatalf(" fork %d: failed to insert into chain: %v", i, err)
+		}
+		if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() {
+			t.Errorf(" fork %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
+		}
+	}
+}
+
+// Tests that importing small side forks doesn't leave junk in the trie database
+// cache (which would eventually cause memory issues).
+func TestTrieForkGC(t *testing.T) {
+	// Generate a canonical chain to act as the main dataset
+	engine := ethash.NewFaker()
+
+	db, _ := ethdb.NewMemDatabase()
+	genesis := new(Genesis).MustCommit(db)
+	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
+
+	// Generate a bunch of fork blocks, each side forking from the canonical chain
+	forks := make([]*types.Block, len(blocks))
+	for i := 0; i < len(forks); i++ {
+		parent := genesis
+		if i > 0 {
+			parent = blocks[i-1]
+		}
+		fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) })
+		forks[i] = fork[0]
+	}
+	// Import the canonical and fork chain side by side, forcing the trie cache to cache both
+	diskdb, _ := ethdb.NewMemDatabase()
+	new(Genesis).MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	for i := 0; i < len(blocks); i++ {
+		if _, err := chain.InsertChain(blocks[i : i+1]); err != nil {
+			t.Fatalf("block %d: failed to insert into chain: %v", i, err)
+		}
+		if _, err := chain.InsertChain(forks[i : i+1]); err != nil {
+			t.Fatalf("fork %d: failed to insert into chain: %v", i, err)
+		}
+	}
+	// Dereference all the recent tries and ensure no past trie is left in
+	for i := 0; i < triesInMemory; i++ {
+		chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root(), common.Hash{})
+		chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root(), common.Hash{})
+	}
+	if len(chain.stateCache.TrieDB().Nodes()) > 0 {
+		t.Fatalf("stale tries still alive after garbase collection")
+	}
+}
+
+// Tests that doing large reorgs works even if the state associated with the
+// forking point is not available any more.
+func TestLargeReorgTrieGC(t *testing.T) {
+	// Generate the original common chain segment and the two competing forks
+	engine := ethash.NewFaker()
+
+	db, _ := ethdb.NewMemDatabase()
+	genesis := new(Genesis).MustCommit(db)
+
+	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
+	original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) })
+	competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
+
+	// Import the shared chain and the original canonical one
+	diskdb, _ := ethdb.NewMemDatabase()
+	new(Genesis).MustCommit(diskdb)
+
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	if _, err := chain.InsertChain(shared); err != nil {
+		t.Fatalf("failed to insert shared chain: %v", err)
+	}
+	if _, err := chain.InsertChain(original); err != nil {
+		t.Fatalf("failed to insert shared chain: %v", err)
+	}
+	// Ensure that the state associated with the forking point is pruned away
+	if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {
+		t.Fatalf("common-but-old ancestor still cache")
+	}
+	// Import the competitor chain without exceeding the canonical's TD and ensure
+	// we have not processed any of the blocks (protection against malicious blocks)
+	if _, err := chain.InsertChain(competitor[:len(competitor)-2]); err != nil {
+		t.Fatalf("failed to insert competitor chain: %v", err)
+	}
+	for i, block := range competitor[:len(competitor)-2] {
+		if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil {
+			t.Fatalf("competitor %d: low TD chain became processed", i)
+		}
+	}
+	// Import the head of the competitor chain, triggering the reorg and ensure we
+	// successfully reprocess all the stashed away blocks.
+	if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil {
+		t.Fatalf("failed to finalize competitor chain: %v", err)
+	}
+	for i, block := range competitor[:len(competitor)-triesInMemory] {
+		if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil {
+			t.Fatalf("competitor %d: competing chain state missing", i)
+		}
+	}
+}

core/chain_indexer.go

@@ -203,6 +203,9 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE
 			if header.ParentHash != prevHash {
 				// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
 				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
+
+				// TODO(karalabe): This operation is expensive and might block, causing the event system to
+				// potentially also lock up. We need to do with on a different thread somehow.
 				if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
 					c.newHead(h.Number.Uint64(), true)
 				}

core/chain_makers.go

@@ -166,7 +166,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
 		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
 		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
-		blockchain, _ := NewBlockChain(db, config, engine, vm.Config{})
+		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{})
 		defer blockchain.Stop()
 
 		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
@@ -192,10 +192,13 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if b.engine != nil {
 			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
 			// Write state changes to db
-			_, err := statedb.CommitTo(db, config.IsEIP158(b.header.Number))
+			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
 			if err != nil {
 				panic(fmt.Sprintf("state write error: %v", err))
 			}
+			if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
+				panic(fmt.Sprintf("trie write error: %v", err))
+			}
 			return block, b.receipts
 		}
 		return nil, nil
@@ -246,7 +249,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B
 	db, _ := ethdb.NewMemDatabase()
 	genesis := gspec.MustCommit(db)
 
-	blockchain, _ := NewBlockChain(db, params.AllEthashProtocolChanges, engine, vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{})
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil

core/chain_makers_test.go

@@ -79,7 +79,7 @@ func ExampleGenerateChain() {
 	})
 
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
 	defer blockchain.Stop()
 
 	if i, err := blockchain.InsertChain(chain); err != nil {

core/dao_test.go

@@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	proConf.DAOForkBlock = forkBlock
 	proConf.DAOForkSupport = true
 
-	proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{})
+	proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{})
 	defer proBc.Stop()
 
 	conDb, _ := ethdb.NewMemDatabase()
@@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	conConf.DAOForkBlock = forkBlock
 	conConf.DAOForkSupport = false
 
-	conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{})
+	conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{})
 	defer conBc.Stop()
 
 	if _, err := proBc.InsertChain(prefix); err != nil {
@@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a pro-fork block, and try to feed into the no-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{})
+		bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
 		defer bc.Stop()
 
 		blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@@ -79,6 +79,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 		}
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
+			t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
+		}
 		blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
 		if _, err := conBc.InsertChain(blocks); err == nil {
 			t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0])
@@ -91,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a no-fork block, and try to feed into the pro-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{})
+		bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
 		defer bc.Stop()
 
 		blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
@@ -101,6 +104,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 		}
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
+			t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
+		}
 		blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
 		if _, err := proBc.InsertChain(blocks); err == nil {
 			t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0])
@@ -114,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{})
+	bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
 	defer bc.Stop()
 
 	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@@ -124,6 +130,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 	}
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
+		t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
+	}
 	blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
 	if _, err := conBc.InsertChain(blocks); err != nil {
 		t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err)
@@ -131,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{})
+	bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
 	defer bc.Stop()
 
 	blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
@@ -141,6 +150,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 	}
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
+		t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
+	}
 	blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
 	if _, err := proBc.InsertChain(blocks); err != nil {
 		t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err)

core/gaspool.go

@@ -44,6 +44,11 @@ func (gp *GasPool) SubGas(amount uint64) error {
 	return nil
 }
 
+// Gas returns the amount of gas remaining in the pool.
+func (gp *GasPool) Gas() uint64 {
+	return uint64(*gp)
+}
+
 func (gp *GasPool) String() string {
 	return fmt.Sprintf("%d", *gp)
 }

core/genesis.go

@@ -169,10 +169,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
 
 	// Check whether the genesis block is already written.
 	if genesis != nil {
-		block, _ := genesis.ToBlock()
+		hash := genesis.ToBlock(nil).Hash()
-		hash := block.Hash()
 		if hash != stored {
-			return genesis.Config, block.Hash(), &GenesisMismatchError{stored, hash}
+			return genesis.Config, hash, &GenesisMismatchError{stored, hash}
 		}
 	}
 
@@ -220,9 +219,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 	}
 }
 
-// ToBlock creates the block and state of a genesis specification.
+// ToBlock creates the genesis block and writes state of a genesis specification
-func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
+// to the given database (or discards it if nil).
-	db, _ := ethdb.NewMemDatabase()
+func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
+	if db == nil {
+		db, _ = ethdb.NewMemDatabase()
+	}
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
 	for addr, account := range g.Alloc {
 		statedb.AddBalance(addr, account.Balance)
@@ -252,19 +254,19 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
 	if g.Difficulty == nil {
 		head.Difficulty = params.GenesisDifficulty
 	}
-	return types.NewBlock(head, nil, nil, nil), statedb
+	statedb.Commit(false)
+	statedb.Database().TrieDB().Commit(root, true)
+
+	return types.NewBlock(head, nil, nil, nil)
 }
 
 // Commit writes the block and state of a genesis specification to the database.
 // The block is committed as the canonical head block.
 func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
-	block, statedb := g.ToBlock()
+	block := g.ToBlock(db)
 	if block.Number().Sign() != 0 {
 		return nil, fmt.Errorf("can't commit genesis block with number > 0")
 	}
-	if _, err := statedb.CommitTo(db, false); err != nil {
-		return nil, fmt.Errorf("cannot write state: %v", err)
-	}
 	if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil {
 		return nil, err
 	}

core/genesis_test.go

@@ -30,11 +30,11 @@ import (
 )
 
 func TestDefaultGenesisBlock(t *testing.T) {
-	block, _ := DefaultGenesisBlock().ToBlock()
+	block := DefaultGenesisBlock().ToBlock(nil)
 	if block.Hash() != params.MainnetGenesisHash {
 		t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash)
 	}
-	block, _ = DefaultTestnetGenesisBlock().ToBlock()
+	block = DefaultTestnetGenesisBlock().ToBlock(nil)
 	if block.Hash() != params.TestnetGenesisHash {
 		t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash)
 	}
@@ -118,7 +118,7 @@ func TestSetupGenesis(t *testing.T) {
 				// Commit the 'old' genesis block with Homestead transition at #2.
 				// Advance to block #4, past the homestead transition block of customg.
 				genesis := oldcustomg.MustCommit(db)
-				bc, _ := NewBlockChain(db, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
+				bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
 				defer bc.Stop()
 				bc.SetValidator(bproc{})
 				bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0))

core/state/database.go

@@ -40,16 +40,23 @@ const (
 
 // Database wraps access to tries and contract code.
 type Database interface {
-	// Accessing tries:
 	// OpenTrie opens the main account trie.
-	// OpenStorageTrie opens the storage trie of an account.
 	OpenTrie(root common.Hash) (Trie, error)
+
+	// OpenStorageTrie opens the storage trie of an account.
 	OpenStorageTrie(addrHash, root common.Hash) (Trie, error)
-	// Accessing contract code:
+
-	ContractCode(addrHash, codeHash common.Hash) ([]byte, error)
-	ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
 	// CopyTrie returns an independent copy of the given trie.
 	CopyTrie(Trie) Trie
+
+	// ContractCode retrieves a particular contract's code.
+	ContractCode(addrHash, codeHash common.Hash) ([]byte, error)
+
+	// ContractCodeSize retrieves a particular contracts code's size.
+	ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
+
+	// TrieDB retrieves the low level trie database used for data storage.
+	TrieDB() *trie.Database
 }
 
 // Trie is a Ethereum Merkle Trie.
@@ -57,26 +64,33 @@ type Trie interface {
 	TryGet(key []byte) ([]byte, error)
 	TryUpdate(key, value []byte) error
 	TryDelete(key []byte) error
-	CommitTo(trie.DatabaseWriter) (common.Hash, error)
+	Commit(onleaf trie.LeafCallback) (common.Hash, error)
 	Hash() common.Hash
 	NodeIterator(startKey []byte) trie.NodeIterator
 	GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed
+	Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory.
+// concurrent use and retains cached trie nodes in memory. The pool is an optional
+// intermediate trie-node memory pool between the low level storage layer and the
+// high level trie abstraction.
 func NewDatabase(db ethdb.Database) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
-	return &cachingDB{db: db, codeSizeCache: csc}
+	return &cachingDB{
+		db:            trie.NewDatabase(db),
+		codeSizeCache: csc,
+	}
 }
 
 type cachingDB struct {
-	db            ethdb.Database
+	db            *trie.Database
 	mu            sync.Mutex
 	pastTries     []*trie.SecureTrie
 	codeSizeCache *lru.Cache
 }
 
+// OpenTrie opens the main account trie.
 func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
@@ -105,10 +119,12 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
 	}
 }
 
+// OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
 	return trie.NewSecure(root, db.db, 0)
 }
 
+// CopyTrie returns an independent copy of the given trie.
 func (db *cachingDB) CopyTrie(t Trie) Trie {
 	switch t := t.(type) {
 	case cachedTrie:
@@ -120,14 +136,16 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 	}
 }
 
+// ContractCode retrieves a particular contract's code.
 func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
-	code, err := db.db.Get(codeHash[:])
+	code, err := db.db.Node(codeHash)
 	if err == nil {
 		db.codeSizeCache.Add(codeHash, len(code))
 	}
 	return code, err
 }
 
+// ContractCodeSize retrieves a particular contracts code's size.
 func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) {
 	if cached, ok := db.codeSizeCache.Get(codeHash); ok {
 		return cached.(int), nil
@@ -139,16 +157,25 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 	return len(code), err
 }
 
+// TrieDB retrieves any intermediate trie-node caching layer.
+func (db *cachingDB) TrieDB() *trie.Database {
+	return db.db
+}
+
 // cachedTrie inserts its trie into a cachingDB on commit.
 type cachedTrie struct {
 	*trie.SecureTrie
 	db *cachingDB
 }
 
-func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
+func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
-	root, err := m.SecureTrie.CommitTo(dbw)
+	root, err := m.SecureTrie.Commit(onleaf)
 	if err == nil {
 		m.db.pushTrie(m.SecureTrie)
 	}
 	return root, err
 }
+
+func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
+	return m.SecureTrie.Prove(key, fromLevel, proofDb)
+}

core/state/iterator_test.go

@@ -21,12 +21,13 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // Tests that the node iterator indeed walks over the entire database contents.
 func TestNodeIteratorCoverage(t *testing.T) {
 	// Create some arbitrary test state to iterate
-	db, mem, root, _ := makeTestState()
+	db, root, _ := makeTestState()
 
 	state, err := New(root, db)
 	if err != nil {
@@ -39,14 +40,18 @@ func TestNodeIteratorCoverage(t *testing.T) {
 			hashes[it.Hash] = struct{}{}
 		}
 	}
-
+	// Cross check the iterated hashes and the database/nodepool content
-	// Cross check the hashes and the database itself
 	for hash := range hashes {
-		if _, err := mem.Get(hash.Bytes()); err != nil {
+		if _, err := db.TrieDB().Node(hash); err != nil {
-			t.Errorf("failed to retrieve reported node %x: %v", hash, err)
+			t.Errorf("failed to retrieve reported node %x", hash)
 		}
 	}
-	for _, key := range mem.Keys() {
+	for _, hash := range db.TrieDB().Nodes() {
+		if _, ok := hashes[hash]; !ok {
+			t.Errorf("state entry not reported %x", hash)
+		}
+	}
+	for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() {
 		if bytes.HasPrefix(key, []byte("secure-key-")) {
 			continue
 		}

core/state/state_object.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie"
 )
 
 var emptyCodeHash = crypto.Keccak256(nil)
@@ -238,12 +237,12 @@ func (self *stateObject) updateRoot(db Database) {
 
 // CommitTrie the storage trie of the object to dwb.
 // This updates the trie root.
-func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error {
+func (self *stateObject) CommitTrie(db Database) error {
 	self.updateTrie(db)
 	if self.dbErr != nil {
 		return self.dbErr
 	}
-	root, err := self.trie.CommitTo(dbw)
+	root, err := self.trie.Commit(nil)
 	if err == nil {
 		self.data.Root = root
 	}

core/state/state_test.go

@@ -48,7 +48,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	// write some of them to the trie
 	s.state.updateStateObject(obj1)
 	s.state.updateStateObject(obj2)
-	s.state.CommitTo(s.db, false)
+	s.state.Commit(false)
 
 	// check that dump contains the state objects that are in trie
 	got := string(s.state.Dump())
@@ -97,7 +97,7 @@ func (s *StateSuite) TestNull(c *checker.C) {
 	//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
 	var value common.Hash
 	s.state.SetState(address, common.Hash{}, value)
-	s.state.CommitTo(s.db, false)
+	s.state.Commit(false)
 	value = s.state.GetState(address, common.Hash{})
 	if !common.EmptyHash(value) {
 		c.Errorf("expected empty hash. got %x", value)
@@ -155,7 +155,7 @@ func TestSnapshot2(t *testing.T) {
 	so0.deleted = false
 	state.setStateObject(so0)
 
-	root, _ := state.CommitTo(db, false)
+	root, _ := state.Commit(false)
 	state.Reset(root)
 
 	// and one with deleted == true

core/state/statedb.go

@@ -36,6 +36,14 @@ type revision struct {
 	journalIndex int
 }
 
+var (
+	// emptyState is the known hash of an empty state trie entry.
+	emptyState = crypto.Keccak256Hash(nil)
+
+	// emptyCode is the known hash of the empty EVM bytecode.
+	emptyCode = crypto.Keccak256Hash(nil)
+)
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -235,6 +243,11 @@ func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
 	return common.Hash{}
 }
 
+// Database retrieves the low level database supporting the lower level trie ops.
+func (self *StateDB) Database() Database {
+	return self.db
+}
+
 // StorageTrie returns the storage trie of an account.
 // The return value is a copy and is nil for non-existent accounts.
 func (self *StateDB) StorageTrie(a common.Address) Trie {
@@ -568,8 +581,8 @@ func (s *StateDB) clearJournalAndRefund() {
 	s.refund = 0
 }
 
-// CommitTo writes the state to the given database.
+// Commit writes the state to the underlying in-memory trie database.
-func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root common.Hash, err error) {
+func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
 	// Commit objects to the trie.
@@ -583,13 +596,11 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
 		case isDirty:
 			// Write any contract code associated with the state object
 			if stateObject.code != nil && stateObject.dirtyCode {
-				if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil {
+				s.db.TrieDB().Insert(common.BytesToHash(stateObject.CodeHash()), stateObject.code)
-					return common.Hash{}, err
-				}
 				stateObject.dirtyCode = false
 			}
 			// Write any storage changes in the state object to its storage trie.
-			if err := stateObject.CommitTrie(s.db, dbw); err != nil {
+			if err := stateObject.CommitTrie(s.db); err != nil {
 				return common.Hash{}, err
 			}
 			// Update the object in the main account trie.
@@ -598,7 +609,20 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
 		delete(s.stateObjectsDirty, addr)
 	}
 	// Write trie changes.
-	root, err = s.trie.CommitTo(dbw)
+	root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error {
+		var account Account
+		if err := rlp.DecodeBytes(leaf, &account); err != nil {
+			return nil
+		}
+		if account.Root != emptyState {
+			s.db.TrieDB().Reference(account.Root, parent)
+		}
+		code := common.BytesToHash(account.CodeHash)
+		if code != emptyCode {
+			s.db.TrieDB().Reference(code, parent)
+		}
+		return nil
+	})
 	log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads())
 	return root, err
 }

core/state/statedb_test.go

@@ -97,10 +97,10 @@ func TestIntermediateLeaks(t *testing.T) {
 	}
 
 	// Commit and cross check the databases.
-	if _, err := transState.CommitTo(transDb, false); err != nil {
+	if _, err := transState.Commit(false); err != nil {
 		t.Fatalf("failed to commit transition state: %v", err)
 	}
-	if _, err := finalState.CommitTo(finalDb, false); err != nil {
+	if _, err := finalState.Commit(false); err != nil {
 		t.Fatalf("failed to commit final state: %v", err)
 	}
 	for _, key := range finalDb.Keys() {
@@ -122,8 +122,8 @@ func TestIntermediateLeaks(t *testing.T) {
 // https://github.com/ethereum/go-ethereum/pull/15549.
 func TestCopy(t *testing.T) {
 	// Create a random state test to copy and modify "independently"
-	mem, _ := ethdb.NewMemDatabase()
+	db, _ := ethdb.NewMemDatabase()
-	orig, _ := New(common.Hash{}, NewDatabase(mem))
+	orig, _ := New(common.Hash{}, NewDatabase(db))
 
 	for i := byte(0); i < 255; i++ {
 		obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@@ -346,11 +346,10 @@ func (test *snapshotTest) run() bool {
 		}
 		action.fn(action, state)
 	}
-
 	// Revert all snapshots in reverse order. Each revert must yield a state
 	// that is equivalent to fresh state with all actions up the snapshot applied.
 	for sindex--; sindex >= 0; sindex-- {
-		checkstate, _ := New(common.Hash{}, NewDatabase(db))
+		checkstate, _ := New(common.Hash{}, state.Database())
 		for _, action := range test.actions[:test.snapshots[sindex]] {
 			action.fn(action, checkstate)
 		}
@@ -409,7 +408,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error {
 
 func (s *StateSuite) TestTouchDelete(c *check.C) {
 	s.state.GetOrNewStateObject(common.Address{})
-	root, _ := s.state.CommitTo(s.db, false)
+	root, _ := s.state.Commit(false)
 	s.state.Reset(root)
 
 	snapshot := s.state.Snapshot()
@@ -417,7 +416,6 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 	if len(s.state.stateObjectsDirty) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
-
 	s.state.RevertToSnapshot(snapshot)
 	if len(s.state.stateObjectsDirty) != 0 {
 		c.Fatal("expected no dirty state object")

core/state/sync_test.go

@@ -36,10 +36,10 @@ type testAccount struct {
 }
 
 // makeTestState create a sample test state to test node-wise reconstruction.
-func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) {
+func makeTestState() (Database, common.Hash, []*testAccount) {
 	// Create an empty state
-	mem, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	db := NewDatabase(mem)
+	db := NewDatabase(diskdb)
 	state, _ := New(common.Hash{}, db)
 
 	// Fill it with some arbitrary data
@@ -61,10 +61,10 @@ func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount)
 		state.updateStateObject(obj)
 		accounts = append(accounts, acc)
 	}
-	root, _ := state.CommitTo(mem, false)
+	root, _ := state.Commit(false)
 
 	// Return the generated state
-	return db, mem, root, accounts
+	return db, root, accounts
 }
 
 // checkStateAccounts cross references a reconstructed state with an expected
@@ -96,7 +96,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
 	if v, _ := db.Get(root[:]); v == nil {
 		return nil // Consider a non existent state consistent.
 	}
-	trie, err := trie.New(root, db)
+	trie, err := trie.New(root, trie.NewDatabase(db))
 	if err != nil {
 		return err
 	}
@@ -138,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T)    { testIterativeStateSync(t,
 
 func testIterativeStateSync(t *testing.T, batch int) {
 	// Create a random state to copy
-	_, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -148,9 +148,9 @@ func testIterativeStateSync(t *testing.T, batch int) {
 	for len(queue) > 0 {
 		results := make([]trie.SyncResult, len(queue))
 		for i, hash := range queue {
-			data, err := srcMem.Get(hash.Bytes())
+			data, err := srcDb.TrieDB().Node(hash)
 			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}
@@ -170,7 +170,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
 // partial results are returned, and the others sent only later.
 func TestIterativeDelayedStateSync(t *testing.T) {
 	// Create a random state to copy
-	_, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -181,9 +181,9 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 		// Sync only half of the scheduled nodes
 		results := make([]trie.SyncResult, len(queue)/2+1)
 		for i, hash := range queue[:len(results)] {
-			data, err := srcMem.Get(hash.Bytes())
+			data, err := srcDb.TrieDB().Node(hash)
 			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}
@@ -207,7 +207,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T)    { testIterativeRandomS
 
 func testIterativeRandomStateSync(t *testing.T, batch int) {
 	// Create a random state to copy
-	_, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -221,9 +221,9 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 		// Fetch all the queued nodes in a random order
 		results := make([]trie.SyncResult, 0, len(queue))
 		for hash := range queue {
-			data, err := srcMem.Get(hash.Bytes())
+			data, err := srcDb.TrieDB().Node(hash)
 			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
 			results = append(results, trie.SyncResult{Hash: hash, Data: data})
 		}
@@ -247,7 +247,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 // partial results are returned (Even those randomly), others sent only later.
 func TestIterativeRandomDelayedStateSync(t *testing.T) {
 	// Create a random state to copy
-	_, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -263,9 +263,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 		for hash := range queue {
 			delete(queue, hash)
 
-			data, err := srcMem.Get(hash.Bytes())
+			data, err := srcDb.TrieDB().Node(hash)
 			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
 			results = append(results, trie.SyncResult{Hash: hash, Data: data})
 
@@ -292,9 +292,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 // the database.
 func TestIncompleteStateSync(t *testing.T) {
 	// Create a random state to copy
-	_, srcMem, srcRoot, srcAccounts := makeTestState()
+	srcDb, srcRoot, srcAccounts := makeTestState()
 
-	checkTrieConsistency(srcMem, srcRoot)
+	checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot)
 
 	// Create a destination state and sync with the scheduler
 	dstDb, _ := ethdb.NewMemDatabase()
@@ -306,9 +306,9 @@ func TestIncompleteStateSync(t *testing.T) {
 		// Fetch a batch of state nodes
 		results := make([]trie.SyncResult, len(queue))
 		for i, hash := range queue {
-			data, err := srcMem.Get(hash.Bytes())
+			data, err := srcDb.TrieDB().Node(hash)
 			if err != nil {
-				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+				t.Fatalf("failed to retrieve node data for %x", hash)
 			}
 			results[i] = trie.SyncResult{Hash: hash, Data: data}
 		}

core/tx_pool_test.go

@@ -78,8 +78,8 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec
 }
 
 func setupTxPool() (*TxPool, *ecdsa.PrivateKey) {
-	db, _ := ethdb.NewMemDatabase()
+	diskdb, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	key, _ := crypto.GenerateKey()

core/types/block.go

@@ -25,6 +25,7 @@ import (
 	"sort"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -121,6 +122,12 @@ func (h *Header) HashNoNonce() common.Hash {
 	})
 }
 
+// Size returns the approximate memory used by all internal contents. It is used
+// to approximate and limit the memory consumption of various caches.
+func (h *Header) Size() common.StorageSize {
+	return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8)
+}
+
 func rlpHash(x interface{}) (h common.Hash) {
 	hw := sha3.NewKeccak256()
 	rlp.Encode(hw, x)
@@ -322,6 +329,8 @@ func (b *Block) HashNoNonce() common.Hash {
 	return b.header.HashNoNonce()
 }
 
+// Size returns the true RLP encoded storage size of the block, either by encoding
+// and returning it, or returning a previsouly cached value.
 func (b *Block) Size() common.StorageSize {
 	if size := b.size.Load(); size != nil {
 		return size.(common.StorageSize)

core/types/receipt.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"unsafe"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -136,6 +137,18 @@ func (r *Receipt) statusEncoding() []byte {
 	return r.PostState
 }
 
+// Size returns the approximate memory used by all internal contents. It is used
+// to approximate and limit the memory consumption of various caches.
+func (r *Receipt) Size() common.StorageSize {
+	size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState))
+
+	size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{}))
+	for _, log := range r.Logs {
+		size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data))
+	}
+	return size
+}
+
 // String implements the Stringer interface.
 func (r *Receipt) String() string {
 	if len(r.PostState) == 0 {

core/types/transaction.go

@@ -206,6 +206,8 @@ func (tx *Transaction) Hash() common.Hash {
 	return v
 }
 
+// Size returns the true RLP encoded storage size of the transaction, either by
+// encoding and returning it, or returning a previsouly cached value.
 func (tx *Transaction) Size() common.StorageSize {
 	if size := tx.size.Load(); size != nil {
 		return size.(common.StorageSize)

core/vm/evm.go

@@ -391,7 +391,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 	return ret, contractAddr, contract.Gas, err
 }
 
-// ChainConfig returns the evmironment's chain configuration
+// ChainConfig returns the environment's chain configuration
 func (evm *EVM) ChainConfig() *params.ChainConfig { return evm.chainConfig }
 
 // Interpreter returns the EVM interpreter

dashboard/assets/.eslintrc

@@ -40,6 +40,9 @@
         'react/jsx-indent':       ['error', 'tab'],
         'react/jsx-indent-props': ['error', 'tab'],
         'react/prefer-stateless-function': 'off',
+        'jsx-quotes':  ['error', 'prefer-single'],
+        'no-plusplus': 'off',
+        'no-console':  ['error', { allow: ['error'] }],
 
         // Specifies the maximum length of a line.
         'max-len': ['warn', 120, 2, {
@@ -49,7 +52,7 @@
             'ignoreStrings':          true,
             'ignoreTemplateLiterals': true,
         }],
-        // Enforces spacing between keys and values in object literal properties.
+        // Enforces consistent spacing between keys and values in object literal properties.
         'key-spacing': ['error', {'align': {
             'beforeColon': false,
             'afterColon':  true,

dashboard/assets/common.jsx

@@ -63,3 +63,9 @@ export type MenuProp = {|...ProvidedMenuProp, id: string|};
 export const MENU: Map<string, {...MenuProp}> = new Map(menuSkeletons.map(({id, menu}) => ([id, {id, ...menu}])));
 
 export const DURATION = 200;
+
+export const styles = {
+	light: {
+		color: 'rgba(255, 255, 255, 0.54)',
+	},
+}

dashboard/assets/components/Body.jsx

@@ -18,35 +18,32 @@
 
 import React, {Component} from 'react';
 
-import withStyles from 'material-ui/styles/withStyles';
-
 import SideBar from './SideBar';
 import Main from './Main';
 import type {Content} from '../types/content';
 
-// Styles for the Body component.
+// styles contains the constant styles of the component.
-const styles = () => ({
+const styles = {
 	body: {
 		display: 'flex',
 		width:   '100%',
 		height:  '100%',
 	},
-});
+};
+
 export type Props = {
-	classes: Object,
 	opened: boolean,
-	changeContent: () => {},
+	changeContent: string => void,
 	active: string,
 	content: Content,
 	shouldUpdate: Object,
 };
+
 // Body renders the body of the dashboard.
 class Body extends Component<Props> {
 	render() {
-		const {classes} = this.props; // The classes property is injected by withStyles().
-
 		return (
-			<div className={classes.body}>
+			<div style={styles.body}>
 				<SideBar
 					opened={this.props.opened}
 					changeContent={this.props.changeContent}
@@ -61,4 +58,4 @@ class Body extends Component<Props> {
 	}
 }
 
-export default withStyles(styles)(Body);
+export default Body;

dashboard/assets/components/ChartRow.jsx

@@ -1,6 +1,6 @@
 // @flow
 
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -17,33 +17,41 @@
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
 import React, {Component} from 'react';
-import type {Node} from 'react';
+import type {ChildrenArray} from 'react';
 
 import Grid from 'material-ui/Grid';
-import {ResponsiveContainer} from 'recharts';
+
+// styles contains the constant styles of the component.
+const styles = {
+	container: {
+		flexWrap: 'nowrap',
+		height:   '100%',
+		maxWidth: '100%',
+		margin:   0,
+	},
+	item: {
+		flex:    1,
+		padding: 0,
+	},
+}
 
 export type Props = {
-	spacing: number,
+	children: ChildrenArray<React$Element<any>>,
-	children: Node,
 };
-// ChartGrid renders a grid container for responsive charts.
+
-// The children are Recharts components extended with the Material-UI's xs property.
+// ChartRow renders a row of equally sized responsive charts.
-class ChartGrid extends Component<Props> {
+class ChartRow extends Component<Props> {
 	render() {
 		return (
-			<Grid container spacing={this.props.spacing}>
+			<Grid container direction='row' style={styles.container} justify='space-between'>
-				{
+				{React.Children.map(this.props.children, child => (
-					React.Children.map(this.props.children, child => (
+					<Grid item xs style={styles.item}>
-						<Grid item xs={child.props.xs}>
+						{child}
-							<ResponsiveContainer width="100%" height={child.props.height}>
+					</Grid>
-								{React.cloneElement(child, {data: child.props.values.map(value => ({value}))})}
+				))}
-							</ResponsiveContainer>
-						</Grid>
-					))
-				}
 			</Grid>
 		);
 	}
 }
 
-export default ChartGrid;
+export default ChartRow;

dashboard/assets/components/CustomTooltip.jsx

@@ -0,0 +1,95 @@
+// @flow
+
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+import React, {Component} from 'react';
+
+import Typography from 'material-ui/Typography';
+import {styles} from '../common';
+
+// multiplier multiplies a number by another.
+export const multiplier = <T>(by: number = 1) => (x: number) => x * by;
+
+// percentPlotter renders a tooltip, which displays the value of the payload followed by a percent sign.
+export const percentPlotter = <T>(text: string, mapper: (T => T) = multiplier(1)) => (payload: T) => {
+	const p = mapper(payload);
+	if (typeof p !== 'number') {
+		return null;
+	}
+	return (
+		<Typography type='caption' color='inherit'>
+			<span style={styles.light}>{text}</span> {p.toFixed(2)} %
+		</Typography>
+	);
+};
+
+// unit contains the units for the bytePlotter.
+const unit = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
+
+// simplifyBytes returns the simplified version of the given value followed by the unit.
+const simplifyBytes = (x: number) => {
+	let i = 0;
+	for (; x > 1024 && i < 5; i++) {
+		x /= 1024;
+	}
+	return x.toFixed(2).toString().concat(' ', unit[i]);
+};
+
+// bytePlotter renders a tooltip, which displays the payload as a byte value.
+export const bytePlotter = <T>(text: string, mapper: (T => T) = multiplier(1)) => (payload: T) => {
+	const p = mapper(payload);
+	if (typeof p !== 'number') {
+		return null;
+	}
+	return (
+		<Typography type='caption' color='inherit'>
+			<span style={styles.light}>{text}</span> {simplifyBytes(p)}
+		</Typography>
+	);
+};
+
+// bytePlotter renders a tooltip, which displays the payload as a byte value followed by '/s'.
+export const bytePerSecPlotter = <T>(text: string, mapper: (T => T) = multiplier(1)) => (payload: T) => {
+	const p = mapper(payload);
+	if (typeof p !== 'number') {
+		return null;
+	}
+	return (
+		<Typography type='caption' color='inherit'>
+			<span style={styles.light}>{text}</span> {simplifyBytes(p)}/s
+		</Typography>
+	);
+};
+
+export type Props = {
+	active: boolean,
+	payload: Object,
+	tooltip: <T>(text: string, mapper?: T => T) => (payload: mixed) => null | React$Element<any>,
+};
+
+// CustomTooltip takes a tooltip function, and uses it to plot the active value of the chart.
+class CustomTooltip extends Component<Props> {
+	render() {
+		const {active, payload, tooltip} = this.props;
+		if (!active || typeof tooltip !== 'function') {
+			return null;
+		}
+		return tooltip(payload[0].value);
+	}
+}
+
+export default CustomTooltip;

dashboard/assets/components/Dashboard.jsx

@@ -22,8 +22,7 @@ import withStyles from 'material-ui/styles/withStyles';
 
 import Header from './Header';
 import Body from './Body';
-import Footer from './Footer';
+import {MENU} from '../common';
-import {MENU} from './Common';
 import type {Content} from '../types/content';
 
 // deepUpdate updates an object corresponding to the given update data, which has
@@ -35,17 +34,17 @@ import type {Content} from '../types/content';
 // the generalization of the message handling. The only necessary thing is to set a
 // handler function for every path of the state in order to maximize the flexibility
 // of the update.
-const deepUpdate = (prev: Object, update: Object, updater: Object) => {
+const deepUpdate = (updater: Object, update: Object, prev: Object): $Shape<Content> => {
 	if (typeof update === 'undefined') {
 		// TODO (kurkomisi): originally this was deep copy, investigate it.
 		return prev;
 	}
 	if (typeof updater === 'function') {
-		return updater(prev, update);
+		return updater(update, prev);
 	}
 	const updated = {};
 	Object.keys(prev).forEach((key) => {
-		updated[key] = deepUpdate(prev[key], update[key], updater[key]);
+		updated[key] = deepUpdate(updater[key], update[key], prev[key]);
 	});
 
 	return updated;
@@ -56,21 +55,25 @@ const deepUpdate = (prev: Object, update: Object, updater: Object) => {
 // whether the involved data was changed or not by checking the message structure.
 //
 // We could return the message itself too, but it's safer not to give access to it.
-const shouldUpdate = (msg: Object, updater: Object) => {
+const shouldUpdate = (updater: Object, msg: Object) => {
 	const su = {};
 	Object.keys(msg).forEach((key) => {
-		su[key] = typeof updater[key] !== 'function' ? shouldUpdate(msg[key], updater[key]) : true;
+		su[key] = typeof updater[key] !== 'function' ? shouldUpdate(updater[key], msg[key]) : true;
 	});
 
 	return su;
 };
 
-// appender is a state update generalization function, which appends the update data
+// replacer is a state updater function, which replaces the original data.
-// to the existing data. limit defines the maximum allowed size of the created array.
+const replacer = <T>(update: T) => update;
-const appender = <T>(limit: number) => (prev: Array<T>, update: Array<T>) => [...prev, ...update].slice(-limit);
 
-// replacer is a state update generalization function, which replaces the original data.
+// appender is a state updater function, which appends the update data to the
-const replacer = <T>(prev: T, update: T) => update;
+// existing data. limit defines the maximum allowed size of the created array,
+// mapper maps the update data.
+const appender = <T>(limit: number, mapper = replacer) => (update: Array<T>, prev: Array<T>) => [
+	...prev,
+	...update.map(sample => mapper(sample)),
+].slice(-limit);
 
 // defaultContent is the initial value of the state content.
 const defaultContent: Content = {
@@ -79,8 +82,14 @@ const defaultContent: Content = {
 		commit:  null,
 	},
 	home: {
-		memory:  [],
+		activeMemory:   [],
-		traffic: [],
+		virtualMemory:  [],
+		networkIngress: [],
+		networkEgress:  [],
+		processCPU:     [],
+		systemCPU:      [],
+		diskRead:       [],
+		diskWrite:      [],
 	},
 	chain:   {},
 	txpool:  {},
@@ -91,16 +100,23 @@ const defaultContent: Content = {
 	},
 };
 
-// updaters contains the state update generalization functions for each path of the state.
+// updaters contains the state updater functions for each path of the state.
-// TODO (kurkomisi): Define a tricky type which embraces the content and the handlers.
+//
+// TODO (kurkomisi): Define a tricky type which embraces the content and the updaters.
 const updaters = {
 	general: {
 		version: replacer,
 		commit:  replacer,
 	},
 	home: {
-		memory:  appender(200),
+		activeMemory:   appender(200),
-		traffic: appender(200),
+		virtualMemory:  appender(200),
+		networkIngress: appender(200),
+		networkEgress:  appender(200),
+		processCPU:     appender(200),
+		systemCPU:      appender(200),
+		diskRead:       appender(200),
+		diskWrite:      appender(200),
 	},
 	chain:   null,
 	txpool:  null,
@@ -111,28 +127,34 @@ const updaters = {
 	},
 };
 
-// styles returns the styles for the Dashboard component.
+// styles contains the constant styles of the component.
-const styles = theme => ({
+const styles = {
+	dashboard: {
+		display:  'flex',
+		flexFlow: 'column',
+		width:    '100%',
+		height:   '100%',
+		zIndex:   1,
+		overflow: 'hidden',
+	}
+};
+
+// themeStyles returns the styles generated from the theme for the component.
+const themeStyles: Object = (theme: Object) => ({
 	dashboard: {
-		display:    'flex',
-		flexFlow:   'column',
-		width:      '100%',
-		height:     '100%',
 		background: theme.palette.background.default,
-		zIndex:     1,
-		overflow:   'hidden',
 	},
 });
 
 export type Props = {
-	classes: Object,
+	classes: Object, // injected by withStyles()
 };
 
 type State = {
 	active: string, // active menu
 	sideBar: boolean, // true if the sidebar is opened
 	content: Content, // the visualized data
-	shouldUpdate: Object // labels for the components, which need to rerender based on the incoming message
+	shouldUpdate: Object, // labels for the components, which need to re-render based on the incoming message
 };
 
 // Dashboard is the main component, which renders the whole page, makes connection with the server and
@@ -176,8 +198,8 @@ class Dashboard extends Component<Props, State> {
 	// update updates the content corresponding to the incoming message.
 	update = (msg: $Shape<Content>) => {
 		this.setState(prevState => ({
-			content:      deepUpdate(prevState.content, msg, updaters),
+			content:      deepUpdate(updaters, msg, prevState.content),
-			shouldUpdate: shouldUpdate(msg, updaters),
+			shouldUpdate: shouldUpdate(updaters, msg),
 		}));
 	};
 
@@ -186,25 +208,17 @@ class Dashboard extends Component<Props, State> {
 		this.setState(prevState => (prevState.active !== newActive ? {active: newActive} : {}));
 	};
 
-	// openSideBar opens the sidebar.
+	// switchSideBar opens or closes the sidebar's state.
-	openSideBar = () => {
+	switchSideBar = () => {
-		this.setState({sideBar: true});
+		this.setState(prevState => ({sideBar: !prevState.sideBar}));
-	};
-
-	// closeSideBar closes the sidebar.
-	closeSideBar = () => {
-		this.setState({sideBar: false});
 	};
 
 	render() {
-		const {classes} = this.props; // The classes property is injected by withStyles().
-
 		return (
-			<div className={classes.dashboard}>
+			<div className={this.props.classes.dashboard} style={styles.dashboard}>
 				<Header
 					opened={this.state.sideBar}
-					openSideBar={this.openSideBar}
+					switchSideBar={this.switchSideBar}
-					closeSideBar={this.closeSideBar}
 				/>
 				<Body
 					opened={this.state.sideBar}
@@ -213,16 +227,9 @@ class Dashboard extends Component<Props, State> {
 					content={this.state.content}
 					shouldUpdate={this.state.shouldUpdate}
 				/>
-				<Footer
-					opened={this.state.sideBar}
-					openSideBar={this.openSideBar}
-					closeSideBar={this.closeSideBar}
-					general={this.state.content.general}
-					shouldUpdate={this.state.shouldUpdate}
-				/>
 			</div>
 		);
 	}
 }
 
-export default withStyles(styles)(Dashboard);
+export default withStyles(themeStyles)(Dashboard);

dashboard/assets/components/Footer.jsx

@@ -19,62 +19,155 @@
 import React, {Component} from 'react';
 
 import withStyles from 'material-ui/styles/withStyles';
-import AppBar from 'material-ui/AppBar';
-import Toolbar from 'material-ui/Toolbar';
 import Typography from 'material-ui/Typography';
+import Grid from 'material-ui/Grid';
+import {ResponsiveContainer, AreaChart, Area, Tooltip} from 'recharts';
 
-import type {General} from '../types/content';
+import ChartRow from './ChartRow';
+import CustomTooltip, {bytePlotter, bytePerSecPlotter, percentPlotter, multiplier} from './CustomTooltip';
+import {styles as commonStyles} from '../common';
+import type {Content} from '../types/content';
 
-// styles contains styles for the Header component.
+// styles contains the constant styles of the component.
-const styles = theme => ({
+const styles = {
+	footer: {
+		maxWidth: '100%',
+		flexWrap: 'nowrap',
+		margin:   0,
+	},
+	chartRowWrapper: {
+		height:  '100%',
+		padding: 0,
+	},
+	doubleChartWrapper: {
+		height:     '100%',
+		width:      '99%',
+		paddingTop: 5,
+	},
+};
+
+// themeStyles returns the styles generated from the theme for the component.
+const themeStyles: Object = (theme: Object) => ({
 	footer: {
 		backgroundColor: theme.palette.background.appBar,
 		color:           theme.palette.getContrastText(theme.palette.background.appBar),
 		zIndex:          theme.zIndex.appBar,
-	},
+		height:          theme.spacing.unit * 10,
-	toolbar: {
-		paddingLeft:    theme.spacing.unit,
-		paddingRight:   theme.spacing.unit,
-		display:        'flex',
-		justifyContent: 'flex-end',
-	},
-	light: {
-		color: 'rgba(255, 255, 255, 0.54)',
 	},
 });
+
 export type Props = {
-	general: General,
+	classes: Object, // injected by withStyles()
-	classes: Object,
+	theme: Object,
+	content: Content,
+	shouldUpdate: Object,
 };
-// TODO (kurkomisi): If the structure is appropriate, make an abstraction of the common parts with the Header.
+
-// Footer renders the header of the dashboard.
+// Footer renders the footer of the dashboard.
 class Footer extends Component<Props> {
 	shouldComponentUpdate(nextProps) {
-		return typeof nextProps.shouldUpdate.logs !== 'undefined';
+		return typeof nextProps.shouldUpdate.home !== 'undefined';
 	}
 
-	info = (about: string, data: string) => (
+	// info renders a label with the given values.
-		<Typography type="caption" color="inherit">
+	info = (about: string, value: ?string) => (value ? (
-			<span className={this.props.classes.light}>{about}</span> {data}
+		<Typography type='caption' color='inherit'>
+			<span style={commonStyles.light}>{about}</span> {value}
 		</Typography>
-	);
+	) : null);
 
-	render() {
+	// doubleChart renders a pair of charts separated by the baseline.
-		const {classes, general} = this.props; // The classes property is injected by withStyles().
+	doubleChart = (syncId, topChart, bottomChart) => {
-		const geth = general.version ? this.info('Geth', general.version) : null;
+		const topKey = 'topKey';
-		const commit = general.commit ? this.info('Commit', general.commit.substring(0, 7)) : null;
+		const bottomKey = 'bottomKey';
+		const topDefault = topChart.default ? topChart.default : 0;
+		const bottomDefault = bottomChart.default ? bottomChart.default : 0;
+		const topTooltip = topChart.tooltip ? (
+			<Tooltip cursor={false} content={<CustomTooltip tooltip={topChart.tooltip} />} />
+		) : null;
+		const bottomTooltip = bottomChart.tooltip ? (
+			<Tooltip cursor={false} content={<CustomTooltip tooltip={bottomChart.tooltip} />} />
+		) : null;
+		const topColor = '#8884d8';
+		const bottomColor = '#82ca9d';
+
+		// Put the samples of the two charts into the same array in order to avoid problems
+		// at the synchronized area charts. If one of the two arrays doesn't have value at
+		// a given position, give it a 0 default value.
+		let data = [...topChart.data.map(({value}) => {
+			const d = {};
+			d[topKey] = value || topDefault;
+			return d;
+		})];
+		for (let i = 0; i < data.length && i < bottomChart.data.length; i++) {
+			// The value needs to be negative in order to plot it upside down.
+			const d = bottomChart.data[i];
+			data[i][bottomKey] = d && d.value ? -d.value : bottomDefault;
+		}
+		data = [...data, ...bottomChart.data.slice(data.length).map(({value}) => {
+			const d = {};
+			d[topKey] = topDefault;
+			d[bottomKey] = -value || bottomDefault;
+			return d;
+		})];
 
 		return (
-			<AppBar position="static" className={classes.footer}>
+			<div style={styles.doubleChartWrapper}>
-				<Toolbar className={classes.toolbar}>
+				<ResponsiveContainer width='100%' height='50%'>
-					<div>
+					<AreaChart data={data} syncId={syncId} >
-						{geth}
+						{topTooltip}
-						{commit}
+						<Area type='monotone' dataKey={topKey} stroke={topColor} fill={topColor} />
-					</div>
+					</AreaChart>
-				</Toolbar>
+				</ResponsiveContainer>
-			</AppBar>
+				<div style={{marginTop: -10, width: '100%', height: '50%'}}>
+					<ResponsiveContainer width='100%' height='100%'>
+						<AreaChart data={data} syncId={syncId} >
+							{bottomTooltip}
+							<Area type='monotone' dataKey={bottomKey} stroke={bottomColor} fill={bottomColor} />
+						</AreaChart>
+					</ResponsiveContainer>
+				</div>
+			</div>
+		);
+	}
+
+	render() {
+		const {content} = this.props;
+		const {general, home} = content;
+
+		return (
+			<Grid container className={this.props.classes.footer} direction='row' alignItems='center' style={styles.footer}>
+				<Grid item xs style={styles.chartRowWrapper}>
+					<ChartRow>
+						{this.doubleChart(
+							'all',
+							{data: home.processCPU, tooltip: percentPlotter('Process')},
+							{data: home.systemCPU, tooltip: percentPlotter('System', multiplier(-1))},
+						)}
+						{this.doubleChart(
+							'all',
+							{data: home.activeMemory, tooltip: bytePlotter('Active')},
+							{data: home.virtualMemory, tooltip: bytePlotter('Virtual', multiplier(-1))},
+						)}
+						{this.doubleChart(
+							'all',
+							{data: home.diskRead, tooltip: bytePerSecPlotter('Disk Read')},
+							{data: home.diskWrite, tooltip: bytePerSecPlotter('Disk Write', multiplier(-1))},
+						)}
+						{this.doubleChart(
+							'all',
+							{data: home.networkIngress, tooltip: bytePerSecPlotter('Download')},
+							{data: home.networkEgress, tooltip: bytePerSecPlotter('Upload', multiplier(-1))},
+						)}
+					</ChartRow>
+				</Grid>
+				<Grid item >
+					{this.info('Geth', general.version)}
+					{this.info('Commit', general.commit ? general.commit.substring(0, 7) : null)}
+				</Grid>
+			</Grid>
 		);
 	}
 }
 
-export default withStyles(styles)(Footer);
+export default withStyles(themeStyles)(Footer);

dashboard/assets/components/Header.jsx

@@ -26,18 +26,22 @@ import IconButton from 'material-ui/IconButton';
 import Typography from 'material-ui/Typography';
 import ChevronLeftIcon from 'material-ui-icons/ChevronLeft';
 
-import {DURATION} from './Common';
+import {DURATION} from '../common';
 
-// arrowDefault is the default style of the arrow button.
+// styles contains the constant styles of the component.
-const arrowDefault = {
+const styles = {
-	transition: `transform ${DURATION}ms`,
+	arrow: {
+		default: {
+			transition: `transform ${DURATION}ms`,
+		},
+		transition: {
+			entered: {transform: 'rotate(180deg)'},
+		},
+	},
 };
-// arrowTransition is the additional style of the arrow button corresponding to the transition's state.
+
-const arrowTransition = {
+// themeStyles returns the styles generated from the theme for the component.
-	entered: {transform: 'rotate(180deg)'},
+const themeStyles = (theme: Object) => ({
-};
-// Styles for the Header component.
-const styles = theme => ({
 	header: {
 		backgroundColor: theme.palette.background.appBar,
 		color:           theme.palette.getContrastText(theme.palette.background.appBar),
@@ -47,53 +51,45 @@ const styles = theme => ({
 		paddingLeft:  theme.spacing.unit,
 		paddingRight: theme.spacing.unit,
 	},
-	mainText: {
+	title: {
 		paddingLeft: theme.spacing.unit,
 	},
 });
+
 export type Props = {
-	classes: Object,
+	classes: Object, // injected by withStyles()
 	opened: boolean,
-	openSideBar: () => {},
+	switchSideBar: () => void,
-	closeSideBar: () => {},
 };
+
 // Header renders the header of the dashboard.
 class Header extends Component<Props> {
 	shouldComponentUpdate(nextProps) {
 		return nextProps.opened !== this.props.opened;
 	}
 
-	// changeSideBar opens or closes the sidebar corresponding to the previous state.
+	// arrow renders a button, which changes the sidebar's state.
-	changeSideBar = () => {
+	arrow = (transitionState: string) => (
-		if (this.props.opened) {
+		<IconButton onClick={this.props.switchSideBar}>
-			this.props.closeSideBar();
-		} else {
-			this.props.openSideBar();
-		}
-	};
-
-	// arrowButton is connected to the sidebar; changes its state.
-	arrowButton = (transitionState: string) => (
-		<IconButton onClick={this.changeSideBar}>
 			<ChevronLeftIcon
 				style={{
-					...arrowDefault,
+					...styles.arrow.default,
-					...arrowTransition[transitionState],
+					...styles.arrow.transition[transitionState],
 				}}
 			/>
 		</IconButton>
 	);
 
 	render() {
-		const {classes, opened} = this.props; // The classes property is injected by withStyles().
+		const {classes, opened} = this.props;
 
 		return (
-			<AppBar position="static" className={classes.header}>
+			<AppBar position='static' className={classes.header}>
 				<Toolbar className={classes.toolbar}>
 					<Transition mountOnEnter in={opened} timeout={{enter: DURATION}}>
-						{this.arrowButton}
+						{this.arrow}
 					</Transition>
-					<Typography type="title" color="inherit" noWrap className={classes.mainText}>
+					<Typography type='title' color='inherit' noWrap className={classes.title}>
 						Go Ethereum Dashboard
 					</Typography>
 				</Toolbar>
@@ -102,4 +98,4 @@ class Header extends Component<Props> {
 	}
 }
 
-export default withStyles(styles)(Header);
+export default withStyles(themeStyles)(Header);

dashboard/assets/components/Home.jsx

@@ -1,77 +0,0 @@
-// @flow
-
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-import React, {Component} from 'react';
-
-import withTheme from 'material-ui/styles/withTheme';
-import {LineChart, AreaChart, Area, YAxis, CartesianGrid, Line} from 'recharts';
-
-import ChartGrid from './ChartGrid';
-import type {ChartEntry} from '../types/content';
-
-export type Props = {
-    theme: Object,
-    memory: Array<ChartEntry>,
-    traffic: Array<ChartEntry>,
-	shouldUpdate: Object,
-};
-// Home renders the home content.
-class Home extends Component<Props> {
-	constructor(props: Props) {
-		super(props);
-		const {theme} = props; // The theme property is injected by withTheme().
-		this.memoryColor = theme.palette.primary[300];
-		this.trafficColor = theme.palette.secondary[300];
-	}
-
-	shouldComponentUpdate(nextProps) {
-		return typeof nextProps.shouldUpdate.home !== 'undefined';
-	}
-
-	memoryColor: Object;
-	trafficColor: Object;
-
-	render() {
-		let {memory, traffic} = this.props;
-		memory = memory.map(({value}) => (value || 0));
-		traffic = traffic.map(({value}) => (value || 0));
-
-		return (
-			<ChartGrid spacing={24}>
-				<AreaChart xs={6} height={300} values={memory}>
-					<YAxis />
-					<Area type="monotone" dataKey="value" stroke={this.memoryColor} fill={this.memoryColor} />
-				</AreaChart>
-				<LineChart xs={6} height={300} values={traffic}>
-					<Line type="monotone" dataKey="value" stroke={this.trafficColor} dot={false} />
-				</LineChart>
-				<LineChart xs={6} height={300} values={memory}>
-					<YAxis />
-					<CartesianGrid stroke="#eee" strokeDasharray="5 5" />
-					<Line type="monotone" dataKey="value" stroke={this.memoryColor} dot={false} />
-				</LineChart>
-				<AreaChart xs={6} height={300} values={traffic}>
-					<CartesianGrid stroke="#eee" strokeDasharray="5 5" vertical={false} />
-					<Area type="monotone" dataKey="value" stroke={this.trafficColor} fill={this.trafficColor} />
-				</AreaChart>
-			</ChartGrid>
-		);
-	}
-}
-
-export default withTheme()(Home);

dashboard/assets/components/Main.jsx

@@ -20,25 +20,38 @@ import React, {Component} from 'react';
 
 import withStyles from 'material-ui/styles/withStyles';
 
-import Home from './Home';
+import {MENU} from '../common';
-import {MENU} from './Common';
+import Footer from './Footer';
 import type {Content} from '../types/content';
 
-// Styles for the Content component.
+// styles contains the constant styles of the component.
-const styles = theme => ({
+const styles = {
+	wrapper: {
+		display:       'flex',
+		flexDirection: 'column',
+		width:         '100%',
+	},
+	content: {
+		flex:     1,
+		overflow: 'auto',
+	},
+};
+
+// themeStyles returns the styles generated from the theme for the component.
+const themeStyles = theme => ({
 	content: {
-		flexGrow:        1,
 		backgroundColor: theme.palette.background.default,
 		padding:         theme.spacing.unit * 3,
-		overflow:        'auto',
 	},
 });
+
 export type Props = {
 	classes: Object,
 	active: string,
 	content: Content,
 	shouldUpdate: Object,
 };
+
 // Main renders the chosen content.
 class Main extends Component<Props> {
 	render() {
@@ -49,8 +62,6 @@ class Main extends Component<Props> {
 		let children = null;
 		switch (active) {
 		case MENU.get('home').id:
-			children = <Home memory={content.home.memory} traffic={content.home.traffic} shouldUpdate={shouldUpdate} />;
-			break;
 		case MENU.get('chain').id:
 		case MENU.get('txpool').id:
 		case MENU.get('network').id:
@@ -61,8 +72,16 @@ class Main extends Component<Props> {
 			children = <div>{content.logs.log.map((log, index) => <div key={index}>{log}</div>)}</div>;
 		}
 
-		return <div className={classes.content}>{children}</div>;
+		return (
+			<div style={styles.wrapper}>
+				<div className={classes.content} style={styles.content}>{children}</div>
+				<Footer
+					content={content}
+					shouldUpdate={shouldUpdate}
+				/>
+			</div>
+		);
 	}
 }
 
-export default withStyles(styles)(Main);
+export default withStyles(themeStyles)(Main);

dashboard/assets/components/SideBar.jsx

@@ -24,18 +24,22 @@ import Icon from 'material-ui/Icon';
 import Transition from 'react-transition-group/Transition';
 import {Icon as FontAwesome} from 'react-fa';
 
-import {MENU, DURATION} from './Common';
+import {MENU, DURATION} from '../common';
 
-// menuDefault is the default style of the menu.
+// styles contains the constant styles of the component.
-const menuDefault = {
+const styles = {
-	transition: `margin-left ${DURATION}ms`,
+	menu: {
+		default: {
+			transition: `margin-left ${DURATION}ms`,
+		},
+		transition: {
+			entered: {marginLeft: -200},
+		},
+	},
 };
-// menuTransition is the additional style of the menu corresponding to the transition's state.
+
-const menuTransition = {
+// themeStyles returns the styles generated from the theme for the component.
-	entered: {marginLeft: -200},
+const themeStyles = theme => ({
-};
-// Styles for the SideBar component.
-const styles = theme => ({
 	list: {
 		background: theme.palette.background.appBar,
 	},
@@ -46,38 +50,32 @@ const styles = theme => ({
 		fontSize: theme.spacing.unit * 3,
 	},
 });
+
 export type Props = {
-	classes: Object,
+	classes: Object, // injected by withStyles()
 	opened: boolean,
-	changeContent: () => {},
+	changeContent: string => void,
 };
+
 // SideBar renders the sidebar of the dashboard.
 class SideBar extends Component<Props> {
-	constructor(props) {
-		super(props);
-
-		// clickOn contains onClick event functions for the menu items.
-		// Instantiate only once, and reuse the existing functions to prevent the creation of
-		// new function instances every time the render method is triggered.
-		this.clickOn = {};
-		MENU.forEach((menu) => {
-			this.clickOn[menu.id] = (event) => {
-				event.preventDefault();
-				props.changeContent(menu.id);
-			};
-		});
-	}
-
 	shouldComponentUpdate(nextProps) {
 		return nextProps.opened !== this.props.opened;
 	}
 
+	// clickOn returns a click event handler function for the given menu item.
+	clickOn = menu => (event) => {
+		event.preventDefault();
+		this.props.changeContent(menu);
+	};
+
+	// menuItems returns the menu items corresponding to the sidebar state.
 	menuItems = (transitionState) => {
 		const {classes} = this.props;
 		const children = [];
 		MENU.forEach((menu) => {
-			children.push(
+			children.push((
-				<ListItem button key={menu.id} onClick={this.clickOn[menu.id]} className={classes.listItem}>
+				<ListItem button key={menu.id} onClick={this.clickOn(menu.id)} className={classes.listItem}>
 					<ListItemIcon>
 						<Icon className={classes.icon}>
 							<FontAwesome name={menu.icon} />
@@ -86,29 +84,25 @@ class SideBar extends Component<Props> {
 					<ListItemText
 						primary={menu.title}
 						style={{
-							...menuDefault,
+							...styles.menu.default,
-							...menuTransition[transitionState],
+							...styles.menu.transition[transitionState],
 							padding: 0,
 						}}
 					/>
-				</ListItem>,
+				</ListItem>
-			);
+			));
 		});
 		return children;
 	};
 
 	// menu renders the list of the menu items.
-	menu = (transitionState) => {
+	menu = (transitionState: Object) => (
-		const {classes} = this.props; // The classes property is injected by withStyles().
+		<div className={this.props.classes.list}>
-
+			<List>
-		return (
+				{this.menuItems(transitionState)}
-			<div className={classes.list}>
+			</List>
-				<List>
+		</div>
-					{this.menuItems(transitionState)}
+	);
-				</List>
-			</div>
-		);
-	};
 
 	render() {
 		return (
@@ -119,4 +113,4 @@ class SideBar extends Component<Props> {
 	}
 }
 
-export default withStyles(styles)(SideBar);
+export default withStyles(themeStyles)(SideBar);

dashboard/assets/index.jsx

@@ -24,18 +24,18 @@ import createMuiTheme from 'material-ui/styles/createMuiTheme';
 
 import Dashboard from './components/Dashboard';
 
-const theme = createMuiTheme({
+const theme: Object = createMuiTheme({
-    palette: {
+	palette: {
-        type: 'dark',
+		type: 'dark',
-    },
+	},
 });
 const dashboard = document.getElementById('dashboard');
 if (dashboard) {
-    // Renders the whole dashboard.
+	// Renders the whole dashboard.
-    render(
+	render(
-        <MuiThemeProvider theme={theme}>
+		<MuiThemeProvider theme={theme}>
-            <Dashboard />
+			<Dashboard />
-        </MuiThemeProvider>,
+		</MuiThemeProvider>,
-        dashboard,
+		dashboard,
-    );
+	);
 }

dashboard/assets/package.json

@@ -1,7 +1,7 @@
 {
     "dependencies": {
         "babel-core": "^6.26.0",
-        "babel-eslint": "^8.1.2",
+        "babel-eslint": "^8.2.1",
         "babel-loader": "^7.1.2",
         "babel-plugin-transform-class-properties": "^6.24.1",
         "babel-plugin-transform-decorators-legacy": "^1.3.4",
@@ -12,8 +12,8 @@
         "babel-preset-stage-0": "^6.24.1",
         "babel-runtime": "^6.26.0",
         "classnames": "^2.2.5",
-        "css-loader": "^0.28.8",
+        "css-loader": "^0.28.9",
-        "eslint": "^4.15.0",
+        "eslint": "^4.16.0",
         "eslint-config-airbnb": "^16.1.0",
         "eslint-loader": "^1.9.0",
         "eslint-plugin-import": "^2.8.0",
@@ -24,16 +24,15 @@
         "flow-bin": "^0.63.1",
         "flow-bin-loader": "^1.0.2",
         "flow-typed": "^2.2.3",
-        "material-ui": "^1.0.0-beta.24",
+        "material-ui": "^1.0.0-beta.30",
         "material-ui-icons": "^1.0.0-beta.17",
         "path": "^0.12.7",
         "react": "^16.2.0",
         "react-dom": "^16.2.0",
         "react-fa": "^5.0.0",
         "react-transition-group": "^2.2.1",
-        "recharts": "^1.0.0-beta.7",
+        "recharts": "^1.0.0-beta.9",
         "style-loader": "^0.19.1",
-        "typeface-roboto": "^0.0.50",
         "url": "^0.11.0",
         "url-loader": "^0.6.2",
         "webpack": "^3.10.0"

dashboard/assets/types/content.jsx

@@ -32,8 +32,14 @@ export type General = {
 };
 
 export type Home = {
-	memory: ChartEntries,
+	activeMemory: ChartEntries,
-	traffic: ChartEntries,
+	virtualMemory: ChartEntries,
+	networkIngress: ChartEntries,
+	networkEgress: ChartEntries,
+	processCPU: ChartEntries,
+	systemCPU: ChartEntries,
+	diskRead: ChartEntries,
+	diskWrite: ChartEntries,
 };
 
 export type ChartEntries = Array<ChartEntry>;

dashboard/config.go

@@ -22,7 +22,7 @@ import "time"
 var DefaultConfig = Config{
 	Host:    "localhost",
 	Port:    8080,
-	Refresh: 3 * time.Second,
+	Refresh: 5 * time.Second,
 }
 
 // Config contains the configuration parameters of the dashboard.

dashboard/cpu.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !windows
+
+package dashboard
+
+import (
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// getProcessCPUTime retrieves the process' CPU time since program startup.
+func getProcessCPUTime() float64 {
+	var usage syscall.Rusage
+	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
+		log.Warn("Failed to retrieve CPU time", "err", err)
+		return 0
+	}
+	return float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1000000
+}

dashboard/cpu_windows.go

@@ -0,0 +1,23 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package dashboard
+
+// getProcessCPUTime returns 0 on Windows as there is no system call to resolve
+// the actual process' CPU time.
+func getProcessCPUTime() float64 {
+	return 0
+}

dashboard/dashboard.go

@@ -21,6 +21,8 @@ package dashboard
 //go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/dashboard.html assets/bundle.js
 //go:generate sh -c "sed 's#var _bundleJs#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
 //go:generate sh -c "sed 's#var _dashboardHtml#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
+//go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/public/...
+//go:generate sh -c "sed 's#var _public#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go"
 //go:generate gofmt -w -s assets.go
 
 import (
@@ -29,10 +31,12 @@ import (
 	"net"
 	"net/http"
 	"path/filepath"
+	"runtime"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/elastic/gosigar"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/params"
@@ -42,8 +46,14 @@ import (
 )
 
 const (
-	memorySampleLimit  = 200 // Maximum number of memory data samples
+	activeMemorySampleLimit   = 200 // Maximum number of active memory data samples
-	trafficSampleLimit = 200 // Maximum number of traffic data samples
+	virtualMemorySampleLimit  = 200 // Maximum number of virtual memory data samples
+	networkIngressSampleLimit = 200 // Maximum number of network ingress data samples
+	networkEgressSampleLimit  = 200 // Maximum number of network egress data samples
+	processCPUSampleLimit     = 200 // Maximum number of process cpu data samples
+	systemCPUSampleLimit      = 200 // Maximum number of system cpu data samples
+	diskReadSampleLimit       = 200 // Maximum number of disk read data samples
+	diskWriteSampleLimit      = 200 // Maximum number of disk write data samples
 )
 
 var nextID uint32 // Next connection id
@@ -71,16 +81,35 @@ type client struct {
 
 // New creates a new dashboard instance with the given configuration.
 func New(config *Config, commit string) (*Dashboard, error) {
-	return &Dashboard{
+	now := time.Now()
+	db := &Dashboard{
 		conns:  make(map[uint32]*client),
 		config: config,
 		quit:   make(chan chan error),
 		charts: &HomeMessage{
-			Memory:  ChartEntries{},
+			ActiveMemory:   emptyChartEntries(now, activeMemorySampleLimit, config.Refresh),
-			Traffic: ChartEntries{},
+			VirtualMemory:  emptyChartEntries(now, virtualMemorySampleLimit, config.Refresh),
+			NetworkIngress: emptyChartEntries(now, networkIngressSampleLimit, config.Refresh),
+			NetworkEgress:  emptyChartEntries(now, networkEgressSampleLimit, config.Refresh),
+			ProcessCPU:     emptyChartEntries(now, processCPUSampleLimit, config.Refresh),
+			SystemCPU:      emptyChartEntries(now, systemCPUSampleLimit, config.Refresh),
+			DiskRead:       emptyChartEntries(now, diskReadSampleLimit, config.Refresh),
+			DiskWrite:      emptyChartEntries(now, diskWriteSampleLimit, config.Refresh),
 		},
 		commit: commit,
-	}, nil
+	}
+	return db, nil
+}
+
+// emptyChartEntries returns a ChartEntry array containing limit number of empty samples.
+func emptyChartEntries(t time.Time, limit int, refresh time.Duration) ChartEntries {
+	ce := make(ChartEntries, limit)
+	for i := 0; i < limit; i++ {
+		ce[i] = &ChartEntry{
+			Time: t.Add(-time.Duration(i) * refresh),
+		}
+	}
+	return ce
 }
 
 // Protocols is a meaningless implementation of node.Service.
@@ -215,8 +244,14 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 			Commit:  db.commit,
 		},
 		Home: &HomeMessage{
-			Memory:  db.charts.Memory,
+			ActiveMemory:   db.charts.ActiveMemory,
-			Traffic: db.charts.Traffic,
+			VirtualMemory:  db.charts.VirtualMemory,
+			NetworkIngress: db.charts.NetworkIngress,
+			NetworkEgress:  db.charts.NetworkEgress,
+			ProcessCPU:     db.charts.ProcessCPU,
+			SystemCPU:      db.charts.SystemCPU,
+			DiskRead:       db.charts.DiskRead,
+			DiskWrite:      db.charts.DiskWrite,
 		},
 	}
 	// Start tracking the connection and drop at connection loss.
@@ -241,6 +276,19 @@ func (db *Dashboard) apiHandler(conn *websocket.Conn) {
 // collectData collects the required data to plot on the dashboard.
 func (db *Dashboard) collectData() {
 	defer db.wg.Done()
+	systemCPUUsage := gosigar.Cpu{}
+	systemCPUUsage.Get()
+	var (
+		prevNetworkIngress = metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Count()
+		prevNetworkEgress  = metrics.DefaultRegistry.Get("p2p/OutboundTraffic").(metrics.Meter).Count()
+		prevProcessCPUTime = getProcessCPUTime()
+		prevSystemCPUUsage = systemCPUUsage
+		prevDiskRead       = metrics.DefaultRegistry.Get("eth/db/chaindata/compact/input").(metrics.Meter).Count()
+		prevDiskWrite      = metrics.DefaultRegistry.Get("eth/db/chaindata/compact/output").(metrics.Meter).Count()
+
+		frequency = float64(db.config.Refresh / time.Second)
+		numCPU    = float64(runtime.NumCPU())
+	)
 
 	for {
 		select {
@@ -248,32 +296,84 @@ func (db *Dashboard) collectData() {
 			errc <- nil
 			return
 		case <-time.After(db.config.Refresh):
-			inboundTraffic := metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Rate1()
+			systemCPUUsage.Get()
-			memoryInUse := metrics.DefaultRegistry.Get("system/memory/inuse").(metrics.Meter).Rate1()
+			var (
+				curNetworkIngress = metrics.DefaultRegistry.Get("p2p/InboundTraffic").(metrics.Meter).Count()
+				curNetworkEgress  = metrics.DefaultRegistry.Get("p2p/OutboundTraffic").(metrics.Meter).Count()
+				curProcessCPUTime = getProcessCPUTime()
+				curSystemCPUUsage = systemCPUUsage
+				curDiskRead       = metrics.DefaultRegistry.Get("eth/db/chaindata/compact/input").(metrics.Meter).Count()
+				curDiskWrite      = metrics.DefaultRegistry.Get("eth/db/chaindata/compact/output").(metrics.Meter).Count()
+
+				deltaNetworkIngress = float64(curNetworkIngress - prevNetworkIngress)
+				deltaNetworkEgress  = float64(curNetworkEgress - prevNetworkEgress)
+				deltaProcessCPUTime = curProcessCPUTime - prevProcessCPUTime
+				deltaSystemCPUUsage = systemCPUUsage.Delta(prevSystemCPUUsage)
+				deltaDiskRead       = curDiskRead - prevDiskRead
+				deltaDiskWrite      = curDiskWrite - prevDiskWrite
+			)
+			prevNetworkIngress = curNetworkIngress
+			prevNetworkEgress = curNetworkEgress
+			prevProcessCPUTime = curProcessCPUTime
+			prevSystemCPUUsage = curSystemCPUUsage
+			prevDiskRead = curDiskRead
+			prevDiskWrite = curDiskWrite
+
 			now := time.Now()
-			memory := &ChartEntry{
+
+			var mem runtime.MemStats
+			runtime.ReadMemStats(&mem)
+			activeMemory := &ChartEntry{
 				Time:  now,
-				Value: memoryInUse,
+				Value: float64(mem.Alloc) / frequency,
 			}
-			traffic := &ChartEntry{
+			virtualMemory := &ChartEntry{
 				Time:  now,
-				Value: inboundTraffic,
+				Value: float64(mem.Sys) / frequency,
 			}
-			first := 0
+			networkIngress := &ChartEntry{
-			if len(db.charts.Memory) == memorySampleLimit {
+				Time:  now,
-				first = 1
+				Value: deltaNetworkIngress / frequency,
 			}
-			db.charts.Memory = append(db.charts.Memory[first:], memory)
+			networkEgress := &ChartEntry{
-			first = 0
+				Time:  now,
-			if len(db.charts.Traffic) == trafficSampleLimit {
+				Value: deltaNetworkEgress / frequency,
-				first = 1
 			}
-			db.charts.Traffic = append(db.charts.Traffic[first:], traffic)
+			processCPU := &ChartEntry{
+				Time:  now,
+				Value: deltaProcessCPUTime / frequency / numCPU * 100,
+			}
+			systemCPU := &ChartEntry{
+				Time:  now,
+				Value: float64(deltaSystemCPUUsage.Sys+deltaSystemCPUUsage.User) / frequency / numCPU,
+			}
+			diskRead := &ChartEntry{
+				Time:  now,
+				Value: float64(deltaDiskRead) / frequency,
+			}
+			diskWrite := &ChartEntry{
+				Time:  now,
+				Value: float64(deltaDiskWrite) / frequency,
+			}
+			db.charts.ActiveMemory = append(db.charts.ActiveMemory[1:], activeMemory)
+			db.charts.VirtualMemory = append(db.charts.VirtualMemory[1:], virtualMemory)
+			db.charts.NetworkIngress = append(db.charts.NetworkIngress[1:], networkIngress)
+			db.charts.NetworkEgress = append(db.charts.NetworkEgress[1:], networkEgress)
+			db.charts.ProcessCPU = append(db.charts.ProcessCPU[1:], processCPU)
+			db.charts.SystemCPU = append(db.charts.SystemCPU[1:], systemCPU)
+			db.charts.DiskRead = append(db.charts.DiskRead[1:], diskRead)
+			db.charts.DiskWrite = append(db.charts.DiskRead[1:], diskWrite)
 
 			db.sendToAll(&Message{
 				Home: &HomeMessage{
-					Memory:  ChartEntries{memory},
+					ActiveMemory:   ChartEntries{activeMemory},
-					Traffic: ChartEntries{traffic},
+					VirtualMemory:  ChartEntries{virtualMemory},
+					NetworkIngress: ChartEntries{networkIngress},
+					NetworkEgress:  ChartEntries{networkEgress},
+					ProcessCPU:     ChartEntries{processCPU},
+					SystemCPU:      ChartEntries{systemCPU},
+					DiskRead:       ChartEntries{diskRead},
+					DiskWrite:      ChartEntries{diskWrite},
 				},
 			})
 		}

dashboard/message.go

@@ -34,8 +34,14 @@ type GeneralMessage struct {
 }
 
 type HomeMessage struct {
-	Memory  ChartEntries `json:"memory,omitempty"`
+	ActiveMemory   ChartEntries `json:"activeMemory,omitempty"`
-	Traffic ChartEntries `json:"traffic,omitempty"`
+	VirtualMemory  ChartEntries `json:"virtualMemory,omitempty"`
+	NetworkIngress ChartEntries `json:"networkIngress,omitempty"`
+	NetworkEgress  ChartEntries `json:"networkEgress,omitempty"`
+	ProcessCPU     ChartEntries `json:"processCPU,omitempty"`
+	SystemCPU      ChartEntries `json:"systemCPU,omitempty"`
+	DiskRead       ChartEntries `json:"diskRead,omitempty"`
+	DiskWrite      ChartEntries `json:"diskWrite,omitempty"`
 }
 
 type ChartEntries []*ChartEntry

eth/api.go

@@ -462,11 +462,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
 
-	oldTrie, err := trie.NewSecure(startBlock.Root(), api.eth.chainDb, 0)
+	oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
 	if err != nil {
 		return nil, err
 	}
-	newTrie, err := trie.NewSecure(endBlock.Root(), api.eth.chainDb, 0)
+	newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
 	if err != nil {
 		return nil, err
 	}

eth/api_tracer.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"runtime"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -34,7 +33,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -72,6 +70,7 @@ type txTraceResult struct {
 type blockTraceTask struct {
 	statedb *state.StateDB   // Intermediate state prepped for tracing
 	block   *types.Block     // Block to trace the transactions from
+	rootref common.Hash      // Trie root reference held for this task
 	results []*txTraceResult // Trace results procudes by the task
 }
 
@@ -90,59 +89,6 @@ type txTraceTask struct {
 	index   int            // Transaction offset in the block
 }
 
-// ephemeralDatabase is a memory wrapper around a proper database, which acts as
-// an ephemeral write layer. This construct is used by the chain tracer to write
-// state tries for intermediate blocks without serializing to disk, but at the
-// same time to allow disk fallback for reads that do no hit the memory layer.
-type ephemeralDatabase struct {
-	diskdb ethdb.Database     // Persistent disk database to fall back to with reads
-	memdb  *ethdb.MemDatabase // Ephemeral memory database for primary reads and writes
-}
-
-func (db *ephemeralDatabase) Put(key []byte, value []byte) error { return db.memdb.Put(key, value) }
-func (db *ephemeralDatabase) Delete(key []byte) error            { return errors.New("delete not supported") }
-func (db *ephemeralDatabase) Close()                             { db.memdb.Close() }
-func (db *ephemeralDatabase) NewBatch() ethdb.Batch {
-	return db.memdb.NewBatch()
-}
-func (db *ephemeralDatabase) Has(key []byte) (bool, error) {
-	if has, _ := db.memdb.Has(key); has {
-		return has, nil
-	}
-	return db.diskdb.Has(key)
-}
-func (db *ephemeralDatabase) Get(key []byte) ([]byte, error) {
-	if blob, _ := db.memdb.Get(key); blob != nil {
-		return blob, nil
-	}
-	return db.diskdb.Get(key)
-}
-
-// Prune does a state sync into a new memory write layer and replaces the old one.
-// This allows us to discard entries that are no longer referenced from the current
-// state.
-func (db *ephemeralDatabase) Prune(root common.Hash) {
-	// Pull the still relevant state data into memory
-	sync := state.NewStateSync(root, db.diskdb)
-	for sync.Pending() > 0 {
-		hash := sync.Missing(1)[0]
-
-		// Move the next trie node from the memory layer into a sync struct
-		node, err := db.memdb.Get(hash[:])
-		if err != nil {
-			panic(err) // memdb must have the data
-		}
-		if _, _, err := sync.Process([]trie.SyncResult{{Hash: hash, Data: node}}); err != nil {
-			panic(err) // it's not possible to fail processing a node
-		}
-	}
-	// Discard the old memory layer and write a new one
-	db.memdb, _ = ethdb.NewMemDatabaseWithCap(db.memdb.Len())
-	if _, err := sync.Commit(db); err != nil {
-		panic(err) // writing into a memdb cannot fail
-	}
-}
-
 // TraceChain returns the structured logs created during the execution of EVM
 // between two blocks (excluding start) and returns them as a JSON object.
 func (api *PrivateDebugAPI) TraceChain(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) {
@@ -188,19 +134,15 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 
 	// Ensure we have a valid starting state before doing any work
 	origin := start.NumberU64()
+	database := state.NewDatabase(api.eth.ChainDb())
 
-	memdb, _ := ethdb.NewMemDatabase()
-	db := &ephemeralDatabase{
-		diskdb: api.eth.ChainDb(),
-		memdb:  memdb,
-	}
 	if number := start.NumberU64(); number > 0 {
 		start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
 		if start == nil {
 			return nil, fmt.Errorf("parent block #%d not found", number-1)
 		}
 	}
-	statedb, err := state.New(start.Root(), state.NewDatabase(db))
+	statedb, err := state.New(start.Root(), database)
 	if err != nil {
 		// If the starting state is missing, allow some number of blocks to be reexecuted
 		reexec := defaultTraceReexec
@@ -213,7 +155,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			if start == nil {
 				break
 			}
-			if statedb, err = state.New(start.Root(), state.NewDatabase(db)); err == nil {
+			if statedb, err = state.New(start.Root(), database); err == nil {
 				break
 			}
 		}
@@ -256,7 +198,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 					res, err := api.traceTx(ctx, msg, vmctx, task.statedb, config)
 					if err != nil {
 						task.results[i] = &txTraceResult{Error: err.Error()}
-						log.Warn("Tracing failed", "err", err)
+						log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err)
 						break
 					}
 					task.statedb.DeleteSuicides()
@@ -273,7 +215,6 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 	}
 	// Start a goroutine to feed all the blocks into the tracers
 	begin := time.Now()
-	complete := start.NumberU64()
 
 	go func() {
 		var (
@@ -281,6 +222,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			number uint64
 			traced uint64
 			failed error
+			proot  common.Hash
 		)
 		// Ensure everything is properly cleaned up on any exit path
 		defer func() {
@@ -308,7 +250,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			// Print progress logs if long enough time elapsed
 			if time.Since(logged) > 8*time.Second {
 				if number > origin {
-					log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin))
+					log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin), "memory", database.TrieDB().Size())
 				} else {
 					log.Info("Preparing state for chain trace", "block", number, "start", origin, "elapsed", time.Since(begin))
 				}
@@ -325,13 +267,11 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				txs := block.Transactions()
 
 				select {
-				case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: block, results: make([]*txTraceResult, len(txs))}:
+				case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: block, rootref: proot, results: make([]*txTraceResult, len(txs))}:
 				case <-notifier.Closed():
 					return
 				}
 				traced += uint64(len(txs))
-			} else {
-				atomic.StoreUint64(&complete, number)
 			}
 			// Generate the next state snapshot fast without tracing
 			_, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{})
@@ -340,7 +280,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				break
 			}
 			// Finalize the state so any modifications are written to the trie
-			root, err := statedb.CommitTo(db, true)
+			root, err := statedb.Commit(true)
 			if err != nil {
 				failed = err
 				break
@@ -349,26 +289,14 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				failed = err
 				break
 			}
-			// After every N blocks, prune the database to only retain relevant data
+			// Reference the trie twice, once for us, once for the trancer
-			if (number-start.NumberU64())%4096 == 0 {
+			database.TrieDB().Reference(root, common.Hash{})
-				// Wait until currently pending trace jobs finish
+			if number >= origin {
-				for atomic.LoadUint64(&complete) != number {
+				database.TrieDB().Reference(root, common.Hash{})
-					select {
-					case <-time.After(100 * time.Millisecond):
-					case <-notifier.Closed():
-						return
-					}
-				}
-				// No more concurrent access at this point, prune the database
-				var (
-					nodes = db.memdb.Len()
-					start = time.Now()
-				)
-				db.Prune(root)
-				log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(start))
-
-				statedb, _ = state.New(root, state.NewDatabase(db))
 			}
+			// Dereference all past tries we ourselves are done working with
+			database.TrieDB().Dereference(proot, common.Hash{})
+			proot = root
 		}
 	}()
 
@@ -387,12 +315,14 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 			}
 			done[uint64(result.Block)] = result
 
+			// Dereference any paret tries held in memory by this task
+			database.TrieDB().Dereference(res.rootref, common.Hash{})
+
 			// Stream completed traces to the user, aborting on the first error
 			for result, ok := done[next]; ok; result, ok = done[next] {
 				if len(result.Traces) > 0 || next == end.NumberU64() {
 					notifier.Notify(sub.ID, result)
 				}
-				atomic.StoreUint64(&complete, next)
 				delete(done, next)
 				next++
 			}
@@ -544,18 +474,14 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 	}
 	// Otherwise try to reexec blocks until we find a state or reach our limit
 	origin := block.NumberU64()
+	database := state.NewDatabase(api.eth.ChainDb())
 
-	memdb, _ := ethdb.NewMemDatabase()
-	db := &ephemeralDatabase{
-		diskdb: api.eth.ChainDb(),
-		memdb:  memdb,
-	}
 	for i := uint64(0); i < reexec; i++ {
 		block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
 		if block == nil {
 			break
 		}
-		if statedb, err = state.New(block.Root(), state.NewDatabase(db)); err == nil {
+		if statedb, err = state.New(block.Root(), database); err == nil {
 			break
 		}
 	}
@@ -571,6 +497,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 	var (
 		start  = time.Now()
 		logged time.Time
+		proot  common.Hash
 	)
 	for block.NumberU64() < origin {
 		// Print progress logs if long enough time elapsed
@@ -587,26 +514,18 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 			return nil, err
 		}
 		// Finalize the state so any modifications are written to the trie
-		root, err := statedb.CommitTo(db, true)
+		root, err := statedb.Commit(true)
 		if err != nil {
 			return nil, err
 		}
 		if err := statedb.Reset(root); err != nil {
 			return nil, err
 		}
-		// After every N blocks, prune the database to only retain relevant data
+		database.TrieDB().Reference(root, common.Hash{})
-		if block.NumberU64()%4096 == 0 || block.NumberU64() == origin {
+		database.TrieDB().Dereference(proot, common.Hash{})
-			var (
+		proot = root
-				nodes = db.memdb.Len()
-				begin = time.Now()
-			)
-			db.Prune(root)
-			log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(begin))
-
-			statedb, _ = state.New(root, state.NewDatabase(db))
-		}
 	}
-	log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start))
+	log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", database.TrieDB().Size())
 	return statedb, nil
 }
 

eth/backend.go

@@ -144,9 +144,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 		}
 		core.WriteBlockChainVersion(chainDb, core.BlockChainVersion)
 	}
-
+	var (
-	vmConfig := vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
+		vmConfig    = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
-	eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.engine, vmConfig)
+		cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
+	)
+	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -393,10 +395,10 @@ func (s *Ethereum) Start(srvr *p2p.Server) error {
 	// Figure out a max peers count based on the server limits
 	maxPeers := srvr.MaxPeers
 	if s.config.LightServ > 0 {
-		maxPeers -= s.config.LightPeers
+		if s.config.LightPeers >= srvr.MaxPeers {
-		if maxPeers < srvr.MaxPeers/2 {
+			return fmt.Errorf("invalid peer config: light peer count (%d) >= total peer count (%d)", s.config.LightPeers, srvr.MaxPeers)
-			maxPeers = srvr.MaxPeers / 2
 		}
+		maxPeers -= s.config.LightPeers
 	}
 	// Start the networking layer and the light server if requested
 	s.protocolManager.Start(maxPeers)

eth/bind.go

@@ -1,136 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"context"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/internal/ethapi"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-)
-
-// ContractBackend implements bind.ContractBackend with direct calls to Ethereum
-// internals to support operating on contracts within subprotocols like eth and
-// swarm.
-//
-// Internally this backend uses the already exposed API endpoints of the Ethereum
-// object. These should be rewritten to internal Go method calls when the Go API
-// is refactored to support a clean library use.
-type ContractBackend struct {
-	eapi  *ethapi.PublicEthereumAPI        // Wrapper around the Ethereum object to access metadata
-	bcapi *ethapi.PublicBlockChainAPI      // Wrapper around the blockchain to access chain data
-	txapi *ethapi.PublicTransactionPoolAPI // Wrapper around the transaction pool to access transaction data
-}
-
-// NewContractBackend creates a new native contract backend using an existing
-// Ethereum object.
-func NewContractBackend(apiBackend ethapi.Backend) *ContractBackend {
-	return &ContractBackend{
-		eapi:  ethapi.NewPublicEthereumAPI(apiBackend),
-		bcapi: ethapi.NewPublicBlockChainAPI(apiBackend),
-		txapi: ethapi.NewPublicTransactionPoolAPI(apiBackend, new(ethapi.AddrLocker)),
-	}
-}
-
-// CodeAt retrieves any code associated with the contract from the local API.
-func (b *ContractBackend) CodeAt(ctx context.Context, contract common.Address, blockNum *big.Int) ([]byte, error) {
-	return b.bcapi.GetCode(ctx, contract, toBlockNumber(blockNum))
-}
-
-// CodeAt retrieves any code associated with the contract from the local API.
-func (b *ContractBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
-	return b.bcapi.GetCode(ctx, contract, rpc.PendingBlockNumber)
-}
-
-// ContractCall implements bind.ContractCaller executing an Ethereum contract
-// call with the specified data as the input. The pending flag requests execution
-// against the pending block, not the stable head of the chain.
-func (b *ContractBackend) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNum *big.Int) ([]byte, error) {
-	out, err := b.bcapi.Call(ctx, toCallArgs(msg), toBlockNumber(blockNum))
-	return out, err
-}
-
-// ContractCall implements bind.ContractCaller executing an Ethereum contract
-// call with the specified data as the input. The pending flag requests execution
-// against the pending block, not the stable head of the chain.
-func (b *ContractBackend) PendingCallContract(ctx context.Context, msg ethereum.CallMsg) ([]byte, error) {
-	out, err := b.bcapi.Call(ctx, toCallArgs(msg), rpc.PendingBlockNumber)
-	return out, err
-}
-
-func toCallArgs(msg ethereum.CallMsg) ethapi.CallArgs {
-	args := ethapi.CallArgs{
-		To:   msg.To,
-		From: msg.From,
-		Data: msg.Data,
-		Gas:  hexutil.Uint64(msg.Gas),
-	}
-	if msg.GasPrice != nil {
-		args.GasPrice = hexutil.Big(*msg.GasPrice)
-	}
-	if msg.Value != nil {
-		args.Value = hexutil.Big(*msg.Value)
-	}
-	return args
-}
-
-func toBlockNumber(num *big.Int) rpc.BlockNumber {
-	if num == nil {
-		return rpc.LatestBlockNumber
-	}
-	return rpc.BlockNumber(num.Int64())
-}
-
-// PendingAccountNonce implements bind.ContractTransactor retrieving the current
-// pending nonce associated with an account.
-func (b *ContractBackend) PendingNonceAt(ctx context.Context, account common.Address) (nonce uint64, err error) {
-	out, err := b.txapi.GetTransactionCount(ctx, account, rpc.PendingBlockNumber)
-	if out != nil {
-		nonce = uint64(*out)
-	}
-	return nonce, err
-}
-
-// SuggestGasPrice implements bind.ContractTransactor retrieving the currently
-// suggested gas price to allow a timely execution of a transaction.
-func (b *ContractBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
-	return b.eapi.GasPrice(ctx)
-}
-
-// EstimateGasLimit implements bind.ContractTransactor triing to estimate the gas
-// needed to execute a specific transaction based on the current pending state of
-// the backend blockchain. There is no guarantee that this is the true gas limit
-// requirement as other transactions may be added or removed by miners, but it
-// should provide a basis for setting a reasonable default.
-func (b *ContractBackend) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) {
-	gas, err := b.bcapi.EstimateGas(ctx, toCallArgs(msg))
-	return uint64(gas), err
-}
-
-// SendTransaction implements bind.ContractTransactor injects the transaction
-// into the pending pool for execution.
-func (b *ContractBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error {
-	raw, _ := rlp.EncodeToBytes(tx)
-	_, err := b.txapi.SendRawTransaction(ctx, raw)
-	return err
-}

eth/config.go

@@ -22,6 +22,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -43,8 +44,10 @@ var DefaultConfig = Config{
 		DatasetsOnDisk: 2,
 	},
 	NetworkId:     1,
-	LightPeers:    20,
+	LightPeers:    100,
-	DatabaseCache: 128,
+	DatabaseCache: 768,
+	TrieCache:     256,
+	TrieTimeout:   5 * time.Minute,
 	GasPrice:      big.NewInt(18 * params.Shannon),
 
 	TxPool: core.DefaultTxPoolConfig,
@@ -78,6 +81,7 @@ type Config struct {
 	// Protocol options
 	NetworkId uint64 // Network ID to use for selecting peers to connect to
 	SyncMode  downloader.SyncMode
+	NoPruning bool
 
 	// Light client options
 	LightServ  int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests
@@ -87,6 +91,8 @@ type Config struct {
 	SkipBcVersionCheck bool `toml:"-"`
 	DatabaseHandles    int  `toml:"-"`
 	DatabaseCache      int
+	TrieCache          int
+	TrieTimeout        time.Duration
 
 	// Mining-related options
 	Etherbase    common.Address `toml:",omitempty"`

eth/downloader/downloader.go

@@ -18,10 +18,8 @@
 package downloader
 
 import (
-	"crypto/rand"
 	"errors"
 	"fmt"
-	"math"
 	"math/big"
 	"sync"
 	"sync/atomic"
@@ -61,12 +59,11 @@ var (
 	maxHeadersProcess = 2048      // Number of header download results to import at once into the chain
 	maxResultsProcess = 2048      // Number of content download results to import at once into the chain
 
-	fsHeaderCheckFrequency = 100        // Verification frequency of the downloaded headers during fast sync
+	fsHeaderCheckFrequency = 100             // Verification frequency of the downloaded headers during fast sync
-	fsHeaderSafetyNet      = 2048       // Number of headers to discard in case a chain violation is detected
+	fsHeaderSafetyNet      = 2048            // Number of headers to discard in case a chain violation is detected
-	fsHeaderForceVerify    = 24         // Number of headers to verify before and after the pivot to accept it
+	fsHeaderForceVerify    = 24              // Number of headers to verify before and after the pivot to accept it
-	fsPivotInterval        = 256        // Number of headers out of which to randomize the pivot point
+	fsHeaderContCheck      = 3 * time.Second // Time interval to check for header continuations during state download
-	fsMinFullBlocks        = 64         // Number of blocks to retrieve fully even in fast sync
+	fsMinFullBlocks        = 64              // Number of blocks to retrieve fully even in fast sync
-	fsCriticalTrials       = uint32(32) // Number of times to retry in the cricical section before bailing
 )
 
 var (
@@ -102,9 +99,6 @@ type Downloader struct {
 	peers   *peerSet // Set of active peers from which download can proceed
 	stateDB ethdb.Database
 
-	fsPivotLock  *types.Header // Pivot header on critical section entry (cannot change between retries)
-	fsPivotFails uint32        // Number of subsequent fast sync failures in the critical section
-
 	rttEstimate   uint64 // Round trip time to target for download requests
 	rttConfidence uint64 // Confidence in the estimated RTT (unit: millionths to allow atomic ops)
 
@@ -124,6 +118,7 @@ type Downloader struct {
 	synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing
 	synchronising   int32
 	notified        int32
+	committed       int32
 
 	// Channels
 	headerCh      chan dataPack        // [eth/62] Channel receiving inbound block headers
@@ -156,7 +151,7 @@ type Downloader struct {
 // LightChain encapsulates functions required to synchronise a light chain.
 type LightChain interface {
 	// HasHeader verifies a header's presence in the local chain.
-	HasHeader(h common.Hash, number uint64) bool
+	HasHeader(common.Hash, uint64) bool
 
 	// GetHeaderByHash retrieves a header from the local chain.
 	GetHeaderByHash(common.Hash) *types.Header
@@ -164,8 +159,8 @@ type LightChain interface {
 	// CurrentHeader retrieves the head header from the local chain.
 	CurrentHeader() *types.Header
 
-	// GetTdByHash returns the total difficulty of a local block.
+	// GetTd returns the total difficulty of a local block.
-	GetTdByHash(common.Hash) *big.Int
+	GetTd(common.Hash, uint64) *big.Int
 
 	// InsertHeaderChain inserts a batch of headers into the local chain.
 	InsertHeaderChain([]*types.Header, int) (int, error)
@@ -179,7 +174,7 @@ type BlockChain interface {
 	LightChain
 
 	// HasBlockAndState verifies block and associated states' presence in the local chain.
-	HasBlockAndState(common.Hash) bool
+	HasBlockAndState(common.Hash, uint64) bool
 
 	// GetBlockByHash retrieves a block from the local chain.
 	GetBlockByHash(common.Hash) *types.Block
@@ -271,7 +266,6 @@ func (d *Downloader) Synchronising() bool {
 // RegisterPeer injects a new download peer into the set of block source to be
 // used for fetching hashes and blocks from.
 func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error {
-
 	logger := log.New("peer", id)
 	logger.Trace("Registering sync peer")
 	if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil {
@@ -324,8 +318,13 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode
 		errEmptyHeaderSet, errPeersUnavailable, errTooOld,
 		errInvalidAncestor, errInvalidChain:
 		log.Warn("Synchronisation failed, dropping peer", "peer", id, "err", err)
-		d.dropPeer(id)
+		if d.dropPeer == nil {
-
+			// The dropPeer method is nil when `--copydb` is used for a local copy.
+			// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
+			log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", id)
+		} else {
+			d.dropPeer(id)
+		}
 	default:
 		log.Warn("Synchronisation failed, retrying", "err", err)
 	}
@@ -386,9 +385,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode
 
 	// Set the requested sync mode, unless it's forbidden
 	d.mode = mode
-	if d.mode == FastSync && atomic.LoadUint32(&d.fsPivotFails) >= fsCriticalTrials {
+
-		d.mode = FullSync
-	}
 	// Retrieve the origin peer and initiate the downloading process
 	p := d.peers.Peer(id)
 	if p == nil {
@@ -436,57 +433,40 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
 	d.syncStatsChainHeight = height
 	d.syncStatsLock.Unlock()
 
-	// Initiate the sync using a concurrent header and content retrieval algorithm
+	// Ensure our origin point is below any fast sync pivot point
 	pivot := uint64(0)
-	switch d.mode {
+	if d.mode == FastSync {
-	case LightSync:
+		if height <= uint64(fsMinFullBlocks) {
-		pivot = height
+			origin = 0
-	case FastSync:
-		// Calculate the new fast/slow sync pivot point
-		if d.fsPivotLock == nil {
-			pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval)))
-			if err != nil {
-				panic(fmt.Sprintf("Failed to access crypto random source: %v", err))
-			}
-			if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() {
-				pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64()
-			}
 		} else {
-			// Pivot point locked in, use this and do not pick a new one!
+			pivot = height - uint64(fsMinFullBlocks)
-			pivot = d.fsPivotLock.Number.Uint64()
+			if pivot <= origin {
-		}
-		// If the point is below the origin, move origin back to ensure state download
-		if pivot < origin {
-			if pivot > 0 {
 				origin = pivot - 1
-			} else {
-				origin = 0
 			}
 		}
-		log.Debug("Fast syncing until pivot block", "pivot", pivot)
 	}
-	d.queue.Prepare(origin+1, d.mode, pivot, latest)
+	d.committed = 1
+	if d.mode == FastSync && pivot != 0 {
+		d.committed = 0
+	}
+	// Initiate the sync using a concurrent header and content retrieval algorithm
+	d.queue.Prepare(origin+1, d.mode)
 	if d.syncInitHook != nil {
 		d.syncInitHook(origin, height)
 	}
 
 	fetchers := []func() error{
-		func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved
+		func() error { return d.fetchHeaders(p, origin+1, pivot) }, // Headers are always retrieved
-		func() error { return d.fetchBodies(origin + 1) },   // Bodies are retrieved during normal and fast sync
+		func() error { return d.fetchBodies(origin + 1) },          // Bodies are retrieved during normal and fast sync
-		func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync
+		func() error { return d.fetchReceipts(origin + 1) },        // Receipts are retrieved during fast sync
-		func() error { return d.processHeaders(origin+1, td) },
+		func() error { return d.processHeaders(origin+1, pivot, td) },
 	}
 	if d.mode == FastSync {
 		fetchers = append(fetchers, func() error { return d.processFastSyncContent(latest) })
 	} else if d.mode == FullSync {
 		fetchers = append(fetchers, d.processFullSyncContent)
 	}
-	err = d.spawnSync(fetchers)
+	return d.spawnSync(fetchers)
-	if err != nil && d.mode == FastSync && d.fsPivotLock != nil {
-		// If sync failed in the critical section, bump the fail counter.
-		atomic.AddUint32(&d.fsPivotFails, 1)
-	}
-	return err
 }
 
 // spawnSync runs d.process and all given fetcher functions to completion in
@@ -666,7 +646,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					continue
 				}
 				// Otherwise check if we already know the header or not
-				if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
+				if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
 					number, hash = headers[i].Number.Uint64(), headers[i].Hash()
 
 					// If every header is known, even future ones, the peer straight out lied about its head
@@ -731,7 +711,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				arrived = true
 
 				// Modify the search interval based on the response
-				if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
+				if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
 					end = check
 					break
 				}
@@ -769,7 +749,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 // other peers are only accepted if they map cleanly to the skeleton. If no one
 // can fill in the skeleton - not even the origin peer - it's assumed invalid and
 // the origin is dropped.
-func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
+func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64) error {
 	p.log.Debug("Directing header downloads", "origin", from)
 	defer p.log.Debug("Header download terminated")
 
@@ -820,6 +800,18 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
 			}
 			// If no more headers are inbound, notify the content fetchers and return
 			if packet.Items() == 0 {
+				// Don't abort header fetches while the pivot is downloading
+				if atomic.LoadInt32(&d.committed) == 0 && pivot <= from {
+					p.log.Debug("No headers, waiting for pivot commit")
+					select {
+					case <-time.After(fsHeaderContCheck):
+						getHeaders(from)
+						continue
+					case <-d.cancelCh:
+						return errCancelHeaderFetch
+					}
+				}
+				// Pivot done (or not in fast sync) and no more headers, terminate the process
 				p.log.Debug("No more headers available")
 				select {
 				case d.headerProcCh <- nil:
@@ -853,6 +845,12 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
 			getHeaders(from)
 
 		case <-timeout.C:
+			if d.dropPeer == nil {
+				// The dropPeer method is nil when `--copydb` is used for a local copy.
+				// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
+				p.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", p.id)
+				break
+			}
 			// Header retrieval timed out, consider the peer bad and drop
 			p.log.Debug("Header request timed out", "elapsed", ttl)
 			headerTimeoutMeter.Mark(1)
@@ -1071,7 +1069,13 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
 						setIdle(peer, 0)
 					} else {
 						peer.log.Debug("Stalling delivery, dropping", "type", kind)
-						d.dropPeer(pid)
+						if d.dropPeer == nil {
+							// The dropPeer method is nil when `--copydb` is used for a local copy.
+							// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
+							peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", pid)
+						} else {
+							d.dropPeer(pid)
+						}
 					}
 				}
 			}
@@ -1112,10 +1116,8 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
 				}
 				if request.From > 0 {
 					peer.log.Trace("Requesting new batch of data", "type", kind, "from", request.From)
-				} else if len(request.Headers) > 0 {
-					peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number)
 				} else {
-					peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Hashes))
+					peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number)
 				}
 				// Fetch the chunk and make sure any errors return the hashes to the queue
 				if fetchHook != nil {
@@ -1143,10 +1145,7 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
 // processHeaders takes batches of retrieved headers from an input channel and
 // keeps processing and scheduling them into the header chain and downloader's
 // queue until the stream ends or a failure occurs.
-func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
+func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) error {
-	// Calculate the pivoting point for switching from fast to slow sync
-	pivot := d.queue.FastSyncPivot()
-
 	// Keep a count of uncertain headers to roll back
 	rollback := []*types.Header{}
 	defer func() {
@@ -1171,19 +1170,6 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
 				"header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number),
 				"fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock),
 				"block", fmt.Sprintf("%d->%d", lastBlock, curBlock))
-
-			// If we're already past the pivot point, this could be an attack, thread carefully
-			if rollback[len(rollback)-1].Number.Uint64() > pivot {
-				// If we didn't ever fail, lock in the pivot header (must! not! change!)
-				if atomic.LoadUint32(&d.fsPivotFails) == 0 {
-					for _, header := range rollback {
-						if header.Number.Uint64() == pivot {
-							log.Warn("Fast-sync pivot locked in", "number", pivot, "hash", header.Hash())
-							d.fsPivotLock = header
-						}
-					}
-				}
-			}
 		}
 	}()
 
@@ -1218,7 +1204,8 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
 				// L: Request new headers up from 11 (R's TD was higher, it must have something)
 				// R: Nothing to give
 				if d.mode != LightSync {
-					if !gotHeaders && td.Cmp(d.blockchain.GetTdByHash(d.blockchain.CurrentBlock().Hash())) > 0 {
+					head := d.blockchain.CurrentBlock()
+					if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 {
 						return errStallingPeer
 					}
 				}
@@ -1230,7 +1217,8 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
 				// queued for processing when the header download completes. However, as long as the
 				// peer gave us something useful, we're already happy/progressed (above check).
 				if d.mode == FastSync || d.mode == LightSync {
-					if td.Cmp(d.lightchain.GetTdByHash(d.lightchain.CurrentHeader().Hash())) > 0 {
+					head := d.lightchain.CurrentHeader()
+					if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 {
 						return errStallingPeer
 					}
 				}
@@ -1283,13 +1271,6 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
 						rollback = append(rollback[:0], rollback[len(rollback)-fsHeaderSafetyNet:]...)
 					}
 				}
-				// If we're fast syncing and just pulled in the pivot, make sure it's the one locked in
-				if d.mode == FastSync && d.fsPivotLock != nil && chunk[0].Number.Uint64() <= pivot && chunk[len(chunk)-1].Number.Uint64() >= pivot {
-					if pivot := chunk[int(pivot-chunk[0].Number.Uint64())]; pivot.Hash() != d.fsPivotLock.Hash() {
-						log.Warn("Pivot doesn't match locked in one", "remoteNumber", pivot.Number, "remoteHash", pivot.Hash(), "localNumber", d.fsPivotLock.Number, "localHash", d.fsPivotLock.Hash())
-						return errInvalidChain
-					}
-				}
 				// Unless we're doing light chains, schedule the headers for associated content retrieval
 				if d.mode == FullSync || d.mode == FastSync {
 					// If we've reached the allowed number of pending headers, stall a bit
@@ -1324,7 +1305,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
 // processFullSyncContent takes fetch results from the queue and imports them into the chain.
 func (d *Downloader) processFullSyncContent() error {
 	for {
-		results := d.queue.WaitResults()
+		results := d.queue.Results(true)
 		if len(results) == 0 {
 			return nil
 		}
@@ -1338,30 +1319,28 @@ func (d *Downloader) processFullSyncContent() error {
 }
 
 func (d *Downloader) importBlockResults(results []*fetchResult) error {
-	for len(results) != 0 {
+	// Check for any early termination requests
-		// Check for any termination requests. This makes clean shutdown faster.
+	if len(results) == 0 {
-		select {
+		return nil
-		case <-d.quitCh:
+	}
-			return errCancelContentProcessing
+	select {
-		default:
+	case <-d.quitCh:
-		}
+		return errCancelContentProcessing
-		// Retrieve the a batch of results to import
+	default:
-		items := int(math.Min(float64(len(results)), float64(maxResultsProcess)))
+	}
-		first, last := results[0].Header, results[items-1].Header
+	// Retrieve the a batch of results to import
-		log.Debug("Inserting downloaded chain", "items", len(results),
+	first, last := results[0].Header, results[len(results)-1].Header
-			"firstnum", first.Number, "firsthash", first.Hash(),
+	log.Debug("Inserting downloaded chain", "items", len(results),
-			"lastnum", last.Number, "lasthash", last.Hash(),
+		"firstnum", first.Number, "firsthash", first.Hash(),
-		)
+		"lastnum", last.Number, "lasthash", last.Hash(),
-		blocks := make([]*types.Block, items)
+	)
-		for i, result := range results[:items] {
+	blocks := make([]*types.Block, len(results))
-			blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
+	for i, result := range results {
-		}
+		blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
-		if index, err := d.blockchain.InsertChain(blocks); err != nil {
+	}
-			log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
+	if index, err := d.blockchain.InsertChain(blocks); err != nil {
-			return errInvalidChain
+		log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
-		}
+		return errInvalidChain
-		// Shift the results to the next batch
-		results = results[items:]
 	}
 	return nil
 }
@@ -1369,35 +1348,92 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error {
 // processFastSyncContent takes fetch results from the queue and writes them to the
 // database. It also controls the synchronisation of state nodes of the pivot block.
 func (d *Downloader) processFastSyncContent(latest *types.Header) error {
-	// Start syncing state of the reported head block.
+	// Start syncing state of the reported head block. This should get us most of
-	// This should get us most of the state of the pivot block.
+	// the state of the pivot block.
 	stateSync := d.syncState(latest.Root)
 	defer stateSync.Cancel()
 	go func() {
-		if err := stateSync.Wait(); err != nil {
+		if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
 			d.queue.Close() // wake up WaitResults
 		}
 	}()
-
+	// Figure out the ideal pivot block. Note, that this goalpost may move if the
-	pivot := d.queue.FastSyncPivot()
+	// sync takes long enough for the chain head to move significantly.
+	pivot := uint64(0)
+	if height := latest.Number.Uint64(); height > uint64(fsMinFullBlocks) {
+		pivot = height - uint64(fsMinFullBlocks)
+	}
+	// To cater for moving pivot points, track the pivot block and subsequently
+	// accumulated download results separatey.
+	var (
+		oldPivot *fetchResult   // Locked in pivot block, might change eventually
+		oldTail  []*fetchResult // Downloaded content after the pivot
+	)
 	for {
-		results := d.queue.WaitResults()
+		// Wait for the next batch of downloaded data to be available, and if the pivot
+		// block became stale, move the goalpost
+		results := d.queue.Results(oldPivot == nil) // Block if we're not monitoring pivot staleness
 		if len(results) == 0 {
-			return stateSync.Cancel()
+			// If pivot sync is done, stop
+			if oldPivot == nil {
+				return stateSync.Cancel()
+			}
+			// If sync failed, stop
+			select {
+			case <-d.cancelCh:
+				return stateSync.Cancel()
+			default:
+			}
 		}
 		if d.chainInsertHook != nil {
 			d.chainInsertHook(results)
 		}
+		if oldPivot != nil {
+			results = append(append([]*fetchResult{oldPivot}, oldTail...), results...)
+		}
+		// Split around the pivot block and process the two sides via fast/full sync
+		if atomic.LoadInt32(&d.committed) == 0 {
+			latest = results[len(results)-1].Header
+			if height := latest.Number.Uint64(); height > pivot+2*uint64(fsMinFullBlocks) {
+				log.Warn("Pivot became stale, moving", "old", pivot, "new", height-uint64(fsMinFullBlocks))
+				pivot = height - uint64(fsMinFullBlocks)
+			}
+		}
 		P, beforeP, afterP := splitAroundPivot(pivot, results)
 		if err := d.commitFastSyncData(beforeP, stateSync); err != nil {
 			return err
 		}
 		if P != nil {
-			stateSync.Cancel()
+			// If new pivot block found, cancel old state retrieval and restart
-			if err := d.commitPivotBlock(P); err != nil {
+			if oldPivot != P {
-				return err
+				stateSync.Cancel()
+
+				stateSync = d.syncState(P.Header.Root)
+				defer stateSync.Cancel()
+				go func() {
+					if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
+						d.queue.Close() // wake up WaitResults
+					}
+				}()
+				oldPivot = P
+			}
+			// Wait for completion, occasionally checking for pivot staleness
+			select {
+			case <-stateSync.done:
+				if stateSync.err != nil {
+					return stateSync.err
+				}
+				if err := d.commitPivotBlock(P); err != nil {
+					return err
+				}
+				oldPivot = nil
+
+			case <-time.After(time.Second):
+				oldTail = afterP
+				continue
 			}
 		}
+		// Fast sync done, pivot commit done, full import
 		if err := d.importBlockResults(afterP); err != nil {
 			return err
 		}
@@ -1420,52 +1456,49 @@ func splitAroundPivot(pivot uint64, results []*fetchResult) (p *fetchResult, bef
 }
 
 func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *stateSync) error {
-	for len(results) != 0 {
+	// Check for any early termination requests
-		// Check for any termination requests.
+	if len(results) == 0 {
-		select {
+		return nil
-		case <-d.quitCh:
+	}
-			return errCancelContentProcessing
+	select {
-		case <-stateSync.done:
+	case <-d.quitCh:
-			if err := stateSync.Wait(); err != nil {
+		return errCancelContentProcessing
-				return err
+	case <-stateSync.done:
-			}
+		if err := stateSync.Wait(); err != nil {
-		default:
+			return err
 		}
-		// Retrieve the a batch of results to import
+	default:
-		items := int(math.Min(float64(len(results)), float64(maxResultsProcess)))
+	}
-		first, last := results[0].Header, results[items-1].Header
+	// Retrieve the a batch of results to import
-		log.Debug("Inserting fast-sync blocks", "items", len(results),
+	first, last := results[0].Header, results[len(results)-1].Header
-			"firstnum", first.Number, "firsthash", first.Hash(),
+	log.Debug("Inserting fast-sync blocks", "items", len(results),
-			"lastnumn", last.Number, "lasthash", last.Hash(),
+		"firstnum", first.Number, "firsthash", first.Hash(),
-		)
+		"lastnumn", last.Number, "lasthash", last.Hash(),
-		blocks := make([]*types.Block, items)
+	)
-		receipts := make([]types.Receipts, items)
+	blocks := make([]*types.Block, len(results))
-		for i, result := range results[:items] {
+	receipts := make([]types.Receipts, len(results))
-			blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
+	for i, result := range results {
-			receipts[i] = result.Receipts
+		blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
-		}
+		receipts[i] = result.Receipts
-		if index, err := d.blockchain.InsertReceiptChain(blocks, receipts); err != nil {
+	}
-			log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
+	if index, err := d.blockchain.InsertReceiptChain(blocks, receipts); err != nil {
-			return errInvalidChain
+		log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
-		}
+		return errInvalidChain
-		// Shift the results to the next batch
-		results = results[items:]
 	}
 	return nil
 }
 
 func (d *Downloader) commitPivotBlock(result *fetchResult) error {
-	b := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
+	block := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
-	// Sync the pivot block state. This should complete reasonably quickly because
+	log.Debug("Committing fast sync pivot as new head", "number", block.Number(), "hash", block.Hash())
-	// we've already synced up to the reported head block state earlier.
+	if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{result.Receipts}); err != nil {
-	if err := d.syncState(b.Root()).Wait(); err != nil {
 		return err
 	}
-	log.Debug("Committing fast sync pivot as new head", "number", b.Number(), "hash", b.Hash())
+	if err := d.blockchain.FastSyncCommitHead(block.Hash()); err != nil {
-	if _, err := d.blockchain.InsertReceiptChain([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil {
 		return err
 	}
-	return d.blockchain.FastSyncCommitHead(b.Hash())
+	atomic.StoreInt32(&d.committed, 1)
+	return nil
 }
 
 // DeliverHeaders injects a new batch of block headers received from a remote

eth/downloader/downloader_test.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -45,8 +44,8 @@ var (
 // Reduce some of the parameters to make the tester faster.
 func init() {
 	MaxForkAncestry = uint64(10000)
-	blockCacheLimit = 1024
+	blockCacheItems = 1024
-	fsCriticalTrials = 10
+	fsHeaderContCheck = 500 * time.Millisecond
 }
 
 // downloadTester is a test simulator for mocking out local block chain.
@@ -223,7 +222,7 @@ func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool {
 }
 
 // HasBlockAndState checks if a block and associated state is present in the testers canonical chain.
-func (dl *downloadTester) HasBlockAndState(hash common.Hash) bool {
+func (dl *downloadTester) HasBlockAndState(hash common.Hash, number uint64) bool {
 	block := dl.GetBlockByHash(hash)
 	if block == nil {
 		return false
@@ -293,14 +292,14 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
 func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
 	// For now only check that the state trie is correct
 	if block := dl.GetBlockByHash(hash); block != nil {
-		_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
+		_, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb), 0)
 		return err
 	}
 	return fmt.Errorf("non existent block: %x", hash[:4])
 }
 
-// GetTdByHash retrieves the block's total difficulty from the canonical chain.
+// GetTd retrieves the block's total difficulty from the canonical chain.
-func (dl *downloadTester) GetTdByHash(hash common.Hash) *big.Int {
+func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
@@ -619,28 +618,22 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
 // number of items of the various chain components.
 func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
 	// Initialize the counters for the first fork
-	headers, blocks := lengths[0], lengths[0]
+	headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks
 
-	minReceipts, maxReceipts := lengths[0]-fsMinFullBlocks-fsPivotInterval, lengths[0]-fsMinFullBlocks
+	if receipts < 0 {
-	if minReceipts < 0 {
+		receipts = 1
-		minReceipts = 1
-	}
-	if maxReceipts < 0 {
-		maxReceipts = 1
 	}
 	// Update the counters for each subsequent fork
 	for _, length := range lengths[1:] {
 		headers += length - common
 		blocks += length - common
-
+		receipts += length - common - fsMinFullBlocks
-		minReceipts += length - common - fsMinFullBlocks - fsPivotInterval
-		maxReceipts += length - common - fsMinFullBlocks
 	}
 	switch tester.downloader.mode {
 	case FullSync:
-		minReceipts, maxReceipts = 1, 1
+		receipts = 1
 	case LightSync:
-		blocks, minReceipts, maxReceipts = 1, 1, 1
+		blocks, receipts = 1, 1
 	}
 	if hs := len(tester.ownHeaders); hs != headers {
 		t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers)
@@ -648,11 +641,12 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
 	if bs := len(tester.ownBlocks); bs != blocks {
 		t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks)
 	}
-	if rs := len(tester.ownReceipts); rs < minReceipts || rs > maxReceipts {
+	if rs := len(tester.ownReceipts); rs != receipts {
-		t.Fatalf("synchronised receipts mismatch: have %v, want between [%v, %v]", rs, minReceipts, maxReceipts)
+		t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts)
 	}
 	// Verify the state trie too for fast syncs
-	if tester.downloader.mode == FastSync {
+	/*if tester.downloader.mode == FastSync {
+		pivot := uint64(0)
 		var index int
 		if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common {
 			index = pivot
@@ -660,11 +654,11 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
 			index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
 		}
 		if index > 0 {
-			if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb)); statedb == nil || err != nil {
+			if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil {
 				t.Fatalf("state reconstruction failed: %v", err)
 			}
 		}
-	}
+	}*/
 }
 
 // Tests that simple synchronization against a canonical chain works correctly.
@@ -684,7 +678,7 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@@ -710,7 +704,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a long block chain to download and the tester
-	targetBlocks := 8 * blockCacheLimit
+	targetBlocks := 8 * blockCacheItems
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@@ -745,9 +739,9 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 			cached = len(tester.downloader.queue.blockDonePool)
 			if mode == FastSync {
 				if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached {
-					if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
+					//if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
-						cached = receipts
+					cached = receipts
-					}
+					//}
 				}
 			}
 			frozen = int(atomic.LoadUint32(&blocked))
@@ -755,7 +749,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 			tester.downloader.queue.lock.Unlock()
 			tester.lock.Unlock()
 
-			if cached == blockCacheLimit || retrieved+cached+frozen == targetBlocks+1 {
+			if cached == blockCacheItems || retrieved+cached+frozen == targetBlocks+1 {
 				break
 			}
 		}
@@ -765,8 +759,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 		tester.lock.RLock()
 		retrieved = len(tester.ownBlocks)
 		tester.lock.RUnlock()
-		if cached != blockCacheLimit && retrieved+cached+frozen != targetBlocks+1 {
+		if cached != blockCacheItems && retrieved+cached+frozen != targetBlocks+1 {
-			t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheLimit, retrieved, frozen, targetBlocks+1)
+			t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheItems, retrieved, frozen, targetBlocks+1)
 		}
 		// Permit the blocked blocks to import
 		if atomic.LoadUint32(&blocked) > 0 {
@@ -974,7 +968,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download and the tester
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	if targetBlocks >= MaxHashFetch {
 		targetBlocks = MaxHashFetch - 15
 	}
@@ -1016,12 +1010,12 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) {
 
 	// Create various peers with various parts of the chain
 	targetPeers := 8
-	targetBlocks := targetPeers*blockCacheLimit - 15
+	targetBlocks := targetPeers*blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	for i := 0; i < targetPeers; i++ {
 		id := fmt.Sprintf("peer #%d", i)
-		tester.newPeer(id, protocol, hashes[i*blockCacheLimit:], headers, blocks, receipts)
+		tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts)
 	}
 	if err := tester.sync("peer #0", nil, mode); err != nil {
 		t.Fatalf("failed to synchronise blocks: %v", err)
@@ -1045,7 +1039,7 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Create peers of every type
@@ -1084,7 +1078,7 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a block chain to download
-	targetBlocks := 2*blockCacheLimit - 15
+	targetBlocks := 2*blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@@ -1110,8 +1104,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
 			bodiesNeeded++
 		}
 	}
-	for hash, receipt := range receipts {
+	for _, receipt := range receipts {
-		if mode == FastSync && len(receipt) > 0 && headers[hash].Number.Uint64() <= tester.downloader.queue.fastSyncPivot {
+		if mode == FastSync && len(receipt) > 0 {
 			receiptsNeeded++
 		}
 	}
@@ -1139,7 +1133,7 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Attempt a full sync with an attacker feeding gapped headers
@@ -1174,7 +1168,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Attempt a full sync with an attacker feeding shifted headers
@@ -1208,7 +1202,7 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := 3*fsHeaderSafetyNet + fsPivotInterval + fsMinFullBlocks
+	targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Attempt to sync with an attacker that feeds junk during the fast sync phase.
@@ -1248,7 +1242,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
 	tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
 	missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
 
-	tester.downloader.fsPivotFails = 0
 	tester.downloader.syncInitHook = func(uint64, uint64) {
 		for i := missing; i <= len(hashes); i++ {
 			delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
@@ -1267,8 +1260,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
 			t.Errorf("fast sync pivot block #%d not rolled back", head)
 		}
 	}
-	tester.downloader.fsPivotFails = fsCriticalTrials
-
 	// Synchronise with the valid peer and make sure sync succeeds. Since the last
 	// rollback should also disable fast syncing for this process, verify that we
 	// did a fresh full sync. Note, we can't assert anything about the receipts
@@ -1383,7 +1374,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Set a sync init hook to catch progress changes
@@ -1532,7 +1523,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small enough block chain to download
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
 
 	// Set a sync init hook to catch progress changes
@@ -1609,7 +1600,7 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 	defer tester.terminate()
 
 	// Create a small block chain
-	targetBlocks := blockCacheLimit - 15
+	targetBlocks := blockCacheItems - 15
 	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false)
 
 	// Set a sync init hook to catch progress changes
@@ -1697,6 +1688,7 @@ func TestDeliverHeadersHang(t *testing.T) {
 type floodingTestPeer struct {
 	peer   Peer
 	tester *downloadTester
+	pend   sync.WaitGroup
 }
 
 func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() }
@@ -1717,9 +1709,12 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int
 	deliveriesDone := make(chan struct{}, 500)
 	for i := 0; i < cap(deliveriesDone); i++ {
 		peer := fmt.Sprintf("fake-peer%d", i)
+		ftp.pend.Add(1)
+
 		go func() {
 			ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}})
 			deliveriesDone <- struct{}{}
+			ftp.pend.Done()
 		}()
 	}
 	// Deliver the actual requested headers.
@@ -1751,110 +1746,15 @@ func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) {
 		// Whenever the downloader requests headers, flood it with
 		// a lot of unrequested header deliveries.
 		tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{
-			tester.downloader.peers.peers["peer"].peer,
+			peer:   tester.downloader.peers.peers["peer"].peer,
-			tester,
+			tester: tester,
 		}
 		if err := tester.sync("peer", nil, mode); err != nil {
-			t.Errorf("sync failed: %v", err)
+			t.Errorf("test %d: sync failed: %v", i, err)
 		}
 		tester.terminate()
+
+		// Flush all goroutines to prevent messing with subsequent tests
+		tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait()
 	}
 }
-
-// Tests that if fast sync aborts in the critical section, it can restart a few
-// times before giving up.
-// We use data driven subtests to manage this so that it will be parallel on its own
-// and not with the other tests, avoiding intermittent failures.
-func TestFastCriticalRestarts(t *testing.T) {
-	testCases := []struct {
-		protocol int
-		progress bool
-	}{
-		{63, false},
-		{64, false},
-		{63, true},
-		{64, true},
-	}
-	for _, tc := range testCases {
-		t.Run(fmt.Sprintf("protocol %d progress %v", tc.protocol, tc.progress), func(t *testing.T) {
-			testFastCriticalRestarts(t, tc.protocol, tc.progress)
-		})
-	}
-}
-
-func testFastCriticalRestarts(t *testing.T, protocol int, progress bool) {
-	t.Parallel()
-
-	tester := newTester()
-	defer tester.terminate()
-
-	// Create a large enough blockchin to actually fast sync on
-	targetBlocks := fsMinFullBlocks + 2*fsPivotInterval - 15
-	hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
-
-	// Create a tester peer with a critical section header missing (force failures)
-	tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
-	delete(tester.peerHeaders["peer"], hashes[fsMinFullBlocks-1])
-	tester.downloader.dropPeer = func(id string) {} // We reuse the same "faulty" peer throughout the test
-
-	// Remove all possible pivot state roots and slow down replies (test failure resets later)
-	for i := 0; i < fsPivotInterval; i++ {
-		tester.peerMissingStates["peer"][headers[hashes[fsMinFullBlocks+i]].Root] = true
-	}
-	(tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).setDelay(500 * time.Millisecond) // Enough to reach the critical section
-
-	// Synchronise with the peer a few times and make sure they fail until the retry limit
-	for i := 0; i < int(fsCriticalTrials)-1; i++ {
-		// Attempt a sync and ensure it fails properly
-		if err := tester.sync("peer", nil, FastSync); err == nil {
-			t.Fatalf("failing fast sync succeeded: %v", err)
-		}
-		time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
-
-		// If it's the first failure, pivot should be locked => reenable all others to detect pivot changes
-		if i == 0 {
-			time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
-			if tester.downloader.fsPivotLock == nil {
-				time.Sleep(400 * time.Millisecond) // Make sure the first huge timeout expires too
-				t.Fatalf("pivot block not locked in after critical section failure")
-			}
-			tester.lock.Lock()
-			tester.peerHeaders["peer"][hashes[fsMinFullBlocks-1]] = headers[hashes[fsMinFullBlocks-1]]
-			tester.peerMissingStates["peer"] = map[common.Hash]bool{tester.downloader.fsPivotLock.Root: true}
-			(tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).setDelay(0)
-			tester.lock.Unlock()
-		}
-	}
-	// Return all nodes if we're testing fast sync progression
-	if progress {
-		tester.lock.Lock()
-		tester.peerMissingStates["peer"] = map[common.Hash]bool{}
-		tester.lock.Unlock()
-
-		if err := tester.sync("peer", nil, FastSync); err != nil {
-			t.Fatalf("failed to synchronise blocks in progressed fast sync: %v", err)
-		}
-		time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
-
-		if fails := atomic.LoadUint32(&tester.downloader.fsPivotFails); fails != 1 {
-			t.Fatalf("progressed pivot trial count mismatch: have %v, want %v", fails, 1)
-		}
-		assertOwnChain(t, tester, targetBlocks+1)
-	} else {
-		if err := tester.sync("peer", nil, FastSync); err == nil {
-			t.Fatalf("succeeded to synchronise blocks in failed fast sync")
-		}
-		time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
-
-		if fails := atomic.LoadUint32(&tester.downloader.fsPivotFails); fails != fsCriticalTrials {
-			t.Fatalf("failed pivot trial count mismatch: have %v, want %v", fails, fsCriticalTrials)
-		}
-	}
-	// Retry limit exhausted, downloader will switch to full sync, should succeed
-	if err := tester.sync("peer", nil, FastSync); err != nil {
-		t.Fatalf("failed to synchronise blocks in slow sync: %v", err)
-	}
-	// Note, we can't assert the chain here because the test asserter assumes sync
-	// completed using a single mode of operation, whereas fast-then-slow can result
-	// in arbitrary intermediate state that's not cleanly verifiable.
-}