Merge remote-tracking branch 'refs/remotes/ethereum/master' into v1.6.x

.mailmap

@@ -73,7 +73,7 @@ RJ Catalano <rj@erisindustries.com>
 
 Nchinda Nchinda <nchinda2@gmail.com>
 
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru> <homotopycolimit@users.noreply.github.com>
 
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 
@@ -94,3 +94,22 @@ Nick Johnson <arachnid@notdot.net>
 Henning Diedrich <hd@eonblast.com>
 Henning Diedrich <hd@eonblast.com> Drake Burroughs <wildfyre@hotmail.com>
 
+Felix Lange <fjl@twurst.com>
+Felix Lange <fjl@twurst.com> <fjl@users.noreply.github.com>
+
+Максим Чусовлянов <mchusovlianov@gmail.com>
+
+Louis Holbrook <dev@holbrook.no>
+Louis Holbrook <dev@holbrook.no> <nolash@users.noreply.github.com>
+
+Thomas Bocek <tom@tomp2p.net>
+
+Victor Tran <vu.tran54@gmail.com>
+
+Justin Drake <drakefjustin@gmail.com>
+
+Frank Wang <eternnoir@gmail.com>
+
+Gary Rong <garyrong0905@gmail.com>
+
+Guillaume Nicolas <guin56@gmail.com>

.travis.yml

@@ -7,7 +7,7 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.8
+      go: 1.8.1
       script:
         - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
@@ -16,7 +16,7 @@ matrix:
         - go run build/ci.go install
 
     - os: osx
-      go: 1.8
+      go: 1.8.1
       sudo: required
       script:
         - brew update

AUTHORS

@@ -3,24 +3,30 @@
 Ales Katona <ales@coinbase.com>
 Alex Leverington <alex@ethdev.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+Brian Schroeder <bts@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
 Daniel A. Nagy <nagy.da@gmail.com>
+Diego Siqueira <DiSiqueira@users.noreply.github.com>
 Elliot Shepherd <elliot@identitii.com>
 Enrique Fynn <enriquefynn@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
 Fabian Vogelsteller <fabian@frozeman.de>
 Fabio Berger <fabioberger1991@gmail.com>
 Felix Lange <fjl@twurst.com>
+Frank Wang <eternnoir@gmail.com>
+Gary Rong <garyrong0905@gmail.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guillaume Nicolas <guin56@gmail.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
 Henning Diedrich <hd@eonblast.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
 Jae Kwon <jkwon.work@gmail.com>
+Jamie Pitts <james.pitts@gmail.com>
 Jason Carver <jacarver@linkedin.com>
 Jeff R. Allen <jra@nella.org>
 Jeffrey Wilcke <jeffrey@ethereum.org>
@@ -28,15 +34,20 @@ Jens Agerberg <github@agerberg.me>
 Jonathan Brown <jbrown@bluedroplet.com>
 Joseph Chow <ethereum@outlook.com>
 Justin Clark-Casey <justincc@justincc.org>
+Justin Drake <drakefjustin@gmail.com>
 Kenji Siu <kenji@isuntv.com>
 Kobi Gurkan <kobigurk@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
+Lewis Marshall <lewis@lmars.net>
+Louis Holbrook <dev@holbrook.no>
+Luca Zeug <luclu@users.noreply.github.com>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
 Martin Holst Swende <martin@swende.se>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Micah Zoltu <micah@zoltu.net>
 Nchinda Nchinda <nchinda2@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
@@ -47,17 +58,28 @@ RJ Catalano <rj@erisindustries.com>
 Ramesh Nair <ram@hiddentao.com>
 Ricardo Catalinas Jiménez <r@untroubled.be>
 Rémy Roy <remyroy@remyroy.com>
+Shintaro Kaneko <kaneshin0120@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
 Steven Roose <stevenroose@gmail.com>
 Taylor Gerring <taylor.gerring@gmail.com>
 Thomas Bocek <tom@tomp2p.net>
 Tosh Camille <tochecamille@gmail.com>
+Valentin Wüstholz <wuestholz@users.noreply.github.com>
+Victor Farazdagi <simple.square@gmail.com>
+Victor Tran <vu.tran54@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
 Vincent G <caktux@gmail.com>
 Vitalik Buterin <v@buterin.com>
+Vivek Anand <vivekanand1101@users.noreply.github.com>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
+Zahoor Mohamed <zahoor@zahoor.in>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
+holisticode <holistic.computing@gmail.com>
+ken10100147 <sunhongping@kanjian.com>
+ligi <ligi@ligi.de>
+xiekeyang <xiekeyang@users.noreply.github.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
+Максим Чусовлянов <mchusovlianov@gmail.com>

Makefile

@@ -44,10 +44,10 @@ clean:
 # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.
 
 devtools:
-	go get -u golang.org/x/tools/cmd/stringer
-	go get -u github.com/jteeuwen/go-bindata/go-bindata
-	go get -u github.com/fjl/gencodec
-	go install ./cmd/abigen
+	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
+	env GOBIN= go get -u github.com/jteeuwen/go-bindata/go-bindata
+	env GOBIN= go get -u github.com/fjl/gencodec
+	env GOBIN= go install ./cmd/abigen
 
 # Cross Compilation Targets (xgo)
 

VERSION

@@ -1 +1 @@
-1.6.0
+1.6.1

accounts/abi/bind/backends/simulated.go

@@ -27,6 +27,7 @@ import (
 	"github.com/expanse-org/go-expanse/accounts/abi/bind"
 	"github.com/expanse-org/go-expanse/common"
 	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
@@ -34,7 +35,6 @@ import (
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
@@ -61,7 +61,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
 	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, genesis.Config, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
 	backend.rollback()
 	return backend
@@ -248,7 +248,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	// Execute the call.
 	msg := callmsg{call}
 
-	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain)
+	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
 	vmenv := vm.NewEVM(evmContext, statedb, b.config, vm.Config{})

accounts/keystore/account_cache.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/account_cache_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/keystore.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/keystore_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-%GETH_ARCH%.zip
-  - 7z x go1.8.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.8.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci-notes.md

@@ -5,9 +5,9 @@ for Ubuntu. Packages are built for the all Ubuntu versions which are supported b
 Canonical:
 
 - Trusty Tahr (14.04 LTS)
-- Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
 - Yakkety Yak (16.10)
+- Zesty Zapus (17.04)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,18 +21,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-expanse with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+Add the gophers PPA and install Go 1.8 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.8 devscripts debhelper
 
 Create the source packages:
 
@@ -40,10 +40,11 @@ Create the source packages:
 
 Then go into the source package directory for your running distribution and build the package:
 
-    $ cd dist/expanse-unstable-1.5.0+xenial
+    $ cd dist/expanse-beta-1.5.0+xenial
+
     $ dpkg-buildpackage
 
 Built packages are placed in the dist/ directory.
 
     $ cd ..
-    $ dpkg-deb -c gexp-unstable_1.5.0+xenial_amd64.deb
+    $ dpkg-deb -c gexp-beta_1.6.0+xenial_amd64.deb

build/ci.go

@@ -32,6 +32,7 @@ Available commands are:
    aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
    xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
    xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -73,42 +74,47 @@ var (
 		executablePath("bootnode"),
 		executablePath("evm"),
 		executablePath("gexp"),
-		executablePath("swarm"),
+		executablePath("puppeth"),
 		executablePath("rlpdump"),
+		executablePath("swarm"),
 	}
 
 // A debian package is created for all executables listed here.
 	debExecutables = []debExecutable{
 		{
-			Name:        "gexp",
-			Description: "Expanse CLI client.",
+			Name:        "abigen",
+			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
 		},
 		{
 			Name:        "bootnode",
 			Description: "Ethereum bootnode.",
 		},
-		{
-			Name:        "rlpdump",
-			Description: "Developer utility tool that prints RLP structures.",
-		},
 		{
 			Name:        "evm",
 			Description: "Developer utility version of the EVM (Expanse Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
+		{
+			Name:        "gexp",
+			Description: "Ethereum CLI client.",
+		},
+		{
+			Name:        "puppeth",
+			Description: "Expanse private network manager.",
+		},
+		{
+			Name:        "rlpdump",
+			Description: "Developer utility tool that prints RLP structures.",
+		},
 		{
 			Name:        "swarm",
 			Description: "Expanse Swarm daemon and tools",
 		},
-		{
-			Name:        "abigen",
-			Description: "Source code generator to convert Expanse contract definitions into easy to use, compile-time type-safe Go packages.",
-		},
 	}
 
-// Distros for which packages are created.
-// Note: vivid is unsupported because there is no golang-1.6 package for it.
-// Note: wily is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "yakkety"}
+	// Distros for which packages are created.
+	// Note: vivid is unsupported because there is no golang-1.6 package for it.
+	// Note: wily is unsupported because it was officially deprecated on lanchpad.
+	debDistros = []string{"trusty", "xenial", "yakkety", "zesty"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -146,6 +152,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "purge":
+		doPurge(os.Args[2:])
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -284,7 +292,8 @@ func doTest(cmdline []string) {
 	// Run analysis tools before the tests.
 	build.MustRun(goTool("vet", packages...))
 	if *misspell {
-		spellcheck(packages)
+		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
+		// spellcheck(packages)
 	}
 	// Run the actual tests.
 	gotest := goTool("test", buildFlags(env)...)
@@ -427,6 +436,10 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a PR build")
 		os.Exit(0)
 	}
+	if env.IsCronJob {
+		log.Printf("skipping because this is a cron job")
+		os.Exit(0)
+	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
@@ -952,3 +965,62 @@ func xgoTool(args []string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// Binary distribution cleanups
+
+func doPurge(cmdline []string) {
+	var (
+		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	if env := build.Env(); !env.IsCronJob {
+		log.Printf("skipping because not a cron job")
+		os.Exit(0)
+	}
+	// Create the azure authentication and list the current archives
+	auth := build.AzureBlobstoreConfig{
+		Account:   strings.Split(*store, "/")[0],
+		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+		Container: strings.SplitN(*store, "/", 2)[1],
+	}
+	blobs, err := build.AzureBlobstoreList(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Iterate over the blobs, collect and sort all unstable builds
+	for i := 0; i < len(blobs); i++ {
+		if !strings.Contains(blobs[i].Name, "unstable") {
+			blobs = append(blobs[:i], blobs[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(blobs); i++ {
+		for j := i + 1; j < len(blobs); j++ {
+			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if iTime.After(jTime) {
+				blobs[i], blobs[j] = blobs[j], blobs[i]
+			}
+		}
+	}
+	// Filter out all archives more recent that the given threshold
+	for i, blob := range blobs {
+		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
+			blobs = blobs[:i]
+			break
+		}
+	}
+	// Delete all marked as such and return
+	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
+		log.Fatal(err)
+	}
+}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.7
+Build-Depends: debhelper (>= 8.0.0), golang-1.8
 Standards-Version: 3.9.5
 Homepage: https://expanse.tech
 Vcs-Git: git://github.com/expanse-org/go-expanse.git
@@ -13,7 +13,7 @@ Architecture: any
 Depends: ${misc:Depends}, {{.ExeList}}
 Description: Meta-package to install gexp and other tools
  Meta-package to install gexp and other tools
- 
+
 {{range .Executables}}
 Package: {{$.ExeName .}}
 Conflicts: {{$.ExeConflicts .}}

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/update-license.go

@@ -49,11 +49,13 @@ var (
 		// don't relicense vendored sources
 		"crypto/sha3/", "crypto/ecies/", "log/",
 		"crypto/secp256k1/curve.go",
+		"consensus/ethash/xor.go",
+		"internal/jsre/deps",
+		"cmd/internal/browser",
 		// don't license generated files
 		"contracts/chequebook/contract/",
 		"contracts/ens/contract/",
 		"contracts/release/contract.go",
-		"p2p/discv5/nodeevent_string.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
@@ -284,6 +286,9 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 		if !stat.Mode().IsRegular() {
 			continue
 		}
+		if isGenerated(file) {
+			continue
+		}
 		info, err := fileInfo(file)
 		if err != nil {
 			fmt.Printf("ERROR %s: %v\n", file, err)
@@ -294,6 +299,23 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
+func isGenerated(file string) bool {
+	fd, err := os.Open(file)
+	if err != nil {
+		return false
+	}
+	defer fd.Close()
+	buf := make([]byte, 2048)
+	n, _ := fd.Read(buf)
+	buf = buf[:n]
+	for _, l := range bytes.Split(buf, []byte("\n")) {
+		if bytes.HasPrefix(l, []byte("// Code generated")) {
+			return true
+		}
+	}
+	return false
+}
+
 // fileInfo finds the lowest year in which the given file was committed.
 func fileInfo(file string) (*info, error) {
 	info := &info{file: file, Year: int64(time.Now().Year())}

cmd/evm/disasm.go

@@ -20,10 +20,10 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"strings"
 
 	"github.com/expanse-org/go-expanse/core/asm"
 	cli "gopkg.in/urfave/cli.v1"
-	"strings"
 )
 
 var disasmCommand = cli.Command{

cmd/faucet/faucet.go

@@ -0,0 +1,455 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// faucet is a Ether faucet backed by a light client.
+package main
+
+//go:generate go-bindata -nometadata -o website.go faucet.html
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"math/big"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts"
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/eth"
+	"github.com/expanse-org/go-expanse/eth/downloader"
+	"github.com/expanse-org/go-expanse/ethclient"
+	"github.com/expanse-org/go-expanse/ethstats"
+	"github.com/expanse-org/go-expanse/les"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/node"
+	"github.com/expanse-org/go-expanse/p2p"
+	"github.com/expanse-org/go-expanse/p2p/discover"
+	"github.com/expanse-org/go-expanse/p2p/discv5"
+	"github.com/expanse-org/go-expanse/p2p/nat"
+	"github.com/expanse-org/go-expanse/params"
+	"golang.org/x/net/websocket"
+)
+
+var (
+	genesisFlag = flag.String("genesis", "", "Genesis json file to seed the chain with")
+	apiPortFlag = flag.Int("apiport", 8080, "Listener port for the HTTP API connection")
+	ethPortFlag = flag.Int("ethport", 30303, "Listener port for the devp2p connection")
+	bootFlag    = flag.String("bootnodes", "", "Comma separated bootnode enode URLs to seed with")
+	netFlag     = flag.Int("network", 0, "Network ID to use for the Ethereum protocol")
+	statsFlag   = flag.String("ethstats", "", "Ethstats network monitoring auth string")
+
+	netnameFlag = flag.String("faucet.name", "", "Network name to assign to the faucet")
+	payoutFlag  = flag.Int("faucet.amount", 1, "Number of Ethers to pay out per user request")
+	minutesFlag = flag.Int("faucet.minutes", 1440, "Number of minutes to wait between funding rounds")
+
+	accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with")
+	accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds")
+
+	githubUser  = flag.String("github.user", "", "GitHub user to authenticate with for Gist access")
+	githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with")
+
+	logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+)
+
+var (
+	ether = new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
+)
+
+func main() {
+	// Parse the flags and set up the logger to print everything requested
+	flag.Parse()
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*logFlag), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+
+	// Load up and render the faucet website
+	tmpl, err := Asset("faucet.html")
+	if err != nil {
+		log.Crit("Failed to load the faucet template", "err", err)
+	}
+	period := fmt.Sprintf("%d minute(s)", *minutesFlag)
+	if *minutesFlag%60 == 0 {
+		period = fmt.Sprintf("%d hour(s)", *minutesFlag/60)
+	}
+	website := new(bytes.Buffer)
+	template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{
+		"Network": *netnameFlag,
+		"Amount":  *payoutFlag,
+		"Period":  period,
+	})
+	// Load and parse the genesis block requested by the user
+	blob, err := ioutil.ReadFile(*genesisFlag)
+	if err != nil {
+		log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err)
+	}
+	genesis := new(core.Genesis)
+	if err = json.Unmarshal(blob, genesis); err != nil {
+		log.Crit("Failed to parse genesis block json", "err", err)
+	}
+	// Convert the bootnodes to internal enode representations
+	var enodes []*discv5.Node
+	for _, boot := range strings.Split(*bootFlag, ",") {
+		if url, err := discv5.ParseNode(boot); err == nil {
+			enodes = append(enodes, url)
+		} else {
+			log.Error("Failed to parse bootnode URL", "url", boot, "err", err)
+		}
+	}
+	// Load up the account key and decrypt its password
+	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
+		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
+	}
+	pass := string(blob)
+
+	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
+	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
+		log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err)
+	}
+	acc, err := ks.Import(blob, pass, pass)
+	if err != nil {
+		log.Crit("Failed to import faucet signer account", "err", err)
+	}
+	ks.Unlock(acc, pass)
+
+	// Assemble and start the faucet light service
+	faucet, err := newFaucet(genesis, *ethPortFlag, enodes, *netFlag, *statsFlag, ks, website.Bytes())
+	if err != nil {
+		log.Crit("Failed to start faucet", "err", err)
+	}
+	defer faucet.close()
+
+	if err := faucet.listenAndServe(*apiPortFlag); err != nil {
+		log.Crit("Failed to launch faucet API", "err", err)
+	}
+}
+
+// request represents an accepted funding request.
+type request struct {
+	Username string             `json:"username"` // GitHub user for displaying an avatar
+	Account  common.Address     `json:"account"`  // Ethereum address being funded
+	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
+	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+}
+
+// faucet represents a crypto faucet backed by an Ethereum light client.
+type faucet struct {
+	config *params.ChainConfig // Chain configurations for signing
+	stack  *node.Node          // Ethereum protocol stack
+	client *ethclient.Client   // Client connection to the Ethereum chain
+	index  []byte              // Index page to serve up on the web
+
+	keystore *keystore.KeyStore // Keystore containing the single signer
+	account  accounts.Account   // Account funding user faucet requests
+	nonce    uint64             // Current pending nonce of the faucet
+	price    *big.Int           // Current gas price to issue funds with
+
+	conns   []*websocket.Conn    // Currently live websocket connections
+	history map[string]time.Time // History of users and their funding requests
+	reqs    []*request           // Currently pending funding requests
+	update  chan struct{}        // Channel to signal request updates
+
+	lock sync.RWMutex // Lock protecting the faucet's internals
+}
+
+func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network int, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
+	// Assemble the raw devp2p protocol stack
+	stack, err := node.New(&node.Config{
+		Name:    "geth",
+		Version: params.Version,
+		DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
+		P2P: p2p.Config{
+			NAT:              nat.Any(),
+			NoDiscovery:      true,
+			DiscoveryV5:      true,
+			ListenAddr:       fmt.Sprintf(":%d", port),
+			DiscoveryV5Addr:  fmt.Sprintf(":%d", port+1),
+			MaxPeers:         25,
+			BootstrapNodesV5: enodes,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// Assemble the Ethereum light client protocol
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		cfg := eth.DefaultConfig
+		cfg.SyncMode = downloader.LightSync
+		cfg.NetworkId = network
+		cfg.Genesis = genesis
+		return les.New(ctx, &cfg)
+	}); err != nil {
+		return nil, err
+	}
+	// Assemble the ethstats monitoring and reporting service'
+	if stats != "" {
+		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			var serv *les.LightEthereum
+			ctx.Service(&serv)
+			return ethstats.New(stats, nil, serv)
+		}); err != nil {
+			return nil, err
+		}
+	}
+	// Boot up the client and ensure it connects to bootnodes
+	if err := stack.Start(); err != nil {
+		return nil, err
+	}
+	for _, boot := range enodes {
+		old, _ := discover.ParseNode(boot.String())
+		stack.Server().AddPeer(old)
+	}
+	// Attach to the client and retrieve and interesting metadatas
+	api, err := stack.Attach()
+	if err != nil {
+		stack.Stop()
+		return nil, err
+	}
+	client := ethclient.NewClient(api)
+
+	return &faucet{
+		config:   genesis.Config,
+		stack:    stack,
+		client:   client,
+		index:    index,
+		keystore: ks,
+		account:  ks.Accounts()[0],
+		history:  make(map[string]time.Time),
+		update:   make(chan struct{}, 1),
+	}, nil
+}
+
+// close terminates the Ethereum connection and tears down the faucet.
+func (f *faucet) close() error {
+	return f.stack.Stop()
+}
+
+// listenAndServe registers the HTTP handlers for the faucet and boots it up
+// for service user funding requests.
+func (f *faucet) listenAndServe(port int) error {
+	go f.loop()
+
+	http.HandleFunc("/", f.webHandler)
+	http.Handle("/api", websocket.Handler(f.apiHandler))
+
+	return http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
+}
+
+// webHandler handles all non-api requests, simply flattening and returning the
+// faucet website.
+func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write(f.index)
+}
+
+// apiHandler handles requests for Ether grants and transaction statuses.
+func (f *faucet) apiHandler(conn *websocket.Conn) {
+	// Start tracking the connection and drop at the end
+	f.lock.Lock()
+	f.conns = append(f.conns, conn)
+	f.lock.Unlock()
+
+	defer func() {
+		f.lock.Lock()
+		for i, c := range f.conns {
+			if c == conn {
+				f.conns = append(f.conns[:i], f.conns[i+1:]...)
+				break
+			}
+		}
+		f.lock.Unlock()
+	}()
+	// Send a few initial stats to the client
+	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+	websocket.JSON.Send(conn, map[string]interface{}{
+		"funds":    balance.Div(balance, ether),
+		"funded":   nonce,
+		"peers":    f.stack.Server().PeerCount(),
+		"requests": f.reqs,
+	})
+	header, _ := f.client.HeaderByNumber(context.Background(), nil)
+	websocket.JSON.Send(conn, header)
+
+	// Keep reading requests from the websocket until the connection breaks
+	for {
+		// Fetch the next funding request and validate against github
+		var msg struct {
+			URL string `json:"url"`
+		}
+		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+			return
+		}
+		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
+			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+			continue
+		}
+		log.Info("Faucet funds requested", "gist", msg.URL)
+
+		// Retrieve the gist from the GitHub Gist APIs
+		parts := strings.Split(msg.URL, "/")
+		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+		if *githubUser != "" {
+			req.SetBasicAuth(*githubUser, *githubToken)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		var gist struct {
+			Owner struct {
+				Login string `json:"login"`
+			} `json:"owner"`
+			Files map[string]struct {
+				Content string `json:"content"`
+			} `json:"files"`
+		}
+		err = json.NewDecoder(res.Body).Decode(&gist)
+		res.Body.Close()
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		if gist.Owner.Login == "" {
+			websocket.JSON.Send(conn, map[string]string{"error": "Nice try ;)"})
+			continue
+		}
+		// Iterate over all the files and look for Ethereum addresses
+		var address common.Address
+		for _, file := range gist.Files {
+			if len(file.Content) == 2+common.AddressLength*2 {
+				address = common.HexToAddress(file.Content)
+			}
+		}
+		if address == (common.Address{}) {
+			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
+			continue
+		}
+		// Ensure the user didn't request funds too recently
+		f.lock.Lock()
+		var (
+			fund    bool
+			elapsed time.Duration
+		)
+		if elapsed = time.Since(f.history[gist.Owner.Login]); elapsed > time.Duration(*minutesFlag)*time.Minute {
+			// User wasn't funded recently, create the funding transaction
+			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether), big.NewInt(21000), f.price, nil)
+			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			// Submit the transaction and mark as funded if successful
+			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			f.reqs = append(f.reqs, &request{
+				Username: gist.Owner.Login,
+				Account:  address,
+				Time:     time.Now(),
+				Tx:       signed,
+			})
+			f.history[gist.Owner.Login] = time.Now()
+			fund = true
+		}
+		f.lock.Unlock()
+
+		// Send an error if too frequent funding, othewise a success
+		if !fund {
+			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("User already funded %s ago", common.PrettyDuration(elapsed))})
+			continue
+		}
+		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		select {
+		case f.update <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// loop keeps waiting for interesting events and pushes them out to connected
+// websockets.
+func (f *faucet) loop() {
+	// Wait for chain events and push them to clients
+	heads := make(chan *types.Header, 16)
+	sub, err := f.client.SubscribeNewHead(context.Background(), heads)
+	if err != nil {
+		log.Crit("Failed to subscribe to head events", "err", err)
+	}
+	defer sub.Unsubscribe()
+
+	for {
+		select {
+		case head := <-heads:
+			// New chain head arrived, query the current stats and stream to clients
+			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+			balance = new(big.Int).Div(balance, ether)
+
+			price, _ := f.client.SuggestGasPrice(context.Background())
+			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+			f.lock.Lock()
+			f.price, f.nonce = price, nonce
+			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+				f.reqs = f.reqs[1:]
+			}
+			f.lock.Unlock()
+
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{
+					"funds":    balance,
+					"funded":   f.nonce,
+					"peers":    f.stack.Server().PeerCount(),
+					"requests": f.reqs,
+				}); err != nil {
+					log.Warn("Failed to send stats to client", "err", err)
+					conn.Close()
+					continue
+				}
+				if err := websocket.JSON.Send(conn, head); err != nil {
+					log.Warn("Failed to send header to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+
+		case <-f.update:
+			// Pending requests updated, stream to clients
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+					log.Warn("Failed to send requests to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+		}
+	}
+}

cmd/faucet/faucet.html

@@ -0,0 +1,143 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8">
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+
+		<title>{{.Network}}: GitHub Faucet</title>
+
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
+
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-noty/2.4.1/packaged/jquery.noty.packaged.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.18.0/moment.min.js"></script>
+
+		<style>
+			.vertical-center {
+				min-height: 100%;
+				min-height: 100vh;
+				display: flex;
+				align-items: center;
+			}
+			.progress {
+				position: relative;
+			}
+			.progress span {
+				position: absolute;
+				display: block;
+				width: 100%;
+				color: white;
+			 }
+			 pre {
+				 padding: 6px;
+				 margin: 0;
+			 }
+		</style>
+	</head>
+
+	<body>
+		<div class="vertical-center">
+			<div class="container">
+				<div class="row" style="margin-bottom: 16px;">
+					<div class="col-lg-12">
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+					</div>
+				</div>
+				<div class="row">
+					<div class="col-lg-8 col-lg-offset-2">
+						<div class="input-group">
+							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<span class="input-group-btn">
+								<button class="btn btn-default" type="button" onclick="submit()">Give me Ether!</button>
+							</span>
+						</div>
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+						<div class="col-lg-6 col-lg-offset-3">
+						<div class="panel panel-small panel-default">
+							<div class="panel-body" style="padding: 0; overflow: auto; max-height: 300px;">
+								<table id="requests" class="table table-condensed" style="margin: 0;"></table>
+							</div>
+							<div class="panel-footer">
+								<table style="width: 100%"><tr>
+									<td style="text-align: center;"><i class="fa fa-rss" aria-hidden="true"></i> <span id="peers"></span> peers</td>
+									<td style="text-align: center;"><i class="fa fa-database" aria-hidden="true"></i> <span id="block"></span> blocks</td>
+									<td style="text-align: center;"><i class="fa fa-heartbeat" aria-hidden="true"></i> <span id="funds"></span> Ethers</td>
+									<td style="text-align: center;"><i class="fa fa-university" aria-hidden="true"></i> <span id="funded"></span> funded</td>
+								</tr></table>
+							</div>
+						</div>
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+					<div class="col-lg-12">
+						<h3>How does this work?</h3>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limit of <strong>{{.Amount}} Ether(s) / {{.Period}}</strong>.</p>
+						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<script>
+			// Global variables to hold the current status of the faucet
+			var attempt = 0;
+			var server;
+
+			// Define the function that submits a gist url to the server
+			var submit = function() {
+				server.send(JSON.stringify({url: $("#gist")[0].value}));
+			};
+			// Define a method to reconnect upon server loss
+			var reconnect = function() {
+				if (attempt % 2 == 0) {
+					server = new WebSocket("wss://" + location.host + "/api");
+				} else {
+					server = new WebSocket("ws://" + location.host + "/api");
+				}
+				attempt++;
+
+				server.onmessage = function(event) {
+					var msg = JSON.parse(event.data);
+					if (msg === null) {
+						return;
+					}
+
+					if (msg.funds !== undefined) {
+						$("#funds").text(msg.funds);
+					}
+					if (msg.funded !== undefined) {
+						$("#funded").text(msg.funded);
+					}
+					if (msg.peers !== undefined) {
+						$("#peers").text(msg.peers);
+					}
+					if (msg.number !== undefined) {
+						$("#block").text(parseInt(msg.number, 16));
+					}
+					if (msg.error !== undefined) {
+						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+					}
+					if (msg.success !== undefined) {
+						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+					}
+					if (msg.requests !== undefined && msg.requests !== null) {
+						var content = "";
+						for (var i=0; i<msg.requests.length; i++) {
+							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+						}
+						$("#requests").html("<tbody>" + content + "</tbody>");
+					}
+				}
+				server.onclose = function() { setTimeout(reconnect, 3000); };
+				server.onerror = function() { setTimeout(reconnect, 3000); };
+			}
+			// Establish a websocket connection to the API server
+			reconnect();
+		</script>
+	</body>
+</html>

cmd/gexp/accountcmd.go

@@ -179,8 +179,7 @@ nodes.
 )
 
 func accountList(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-
+	stack, _ := makeConfigNode(ctx)
 	var index int
 	for _, wallet := range stack.AccountManager().Wallets() {
 		for _, account := range wallet.Accounts() {
@@ -278,7 +277,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -296,7 +295,7 @@ func accountUpdate(ctx *cli.Context) error {
 	if len(ctx.Args()) == 0 {
 		utils.Fatalf("No accounts specified to update")
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
 	account, oldPassword := unlockAccount(ctx, ks, ctx.Args().First(), 0, nil)
@@ -317,7 +316,7 @@ func importWallet(ctx *cli.Context) error {
 		utils.Fatalf("Could not read wallet file: %v", err)
 	}
 
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -338,7 +337,7 @@ func accountImport(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("Failed to load the private key: %v", err)
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)

cmd/gexp/chaincmd.go

@@ -244,7 +244,7 @@ func exportChain(ctx *cli.Context) error {
 }
 
 func removeDB(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	dbdir := stack.ResolvePath(utils.ChainDbName(ctx))
 	if !common.FileExist(dbdir) {
 		fmt.Println(dbdir, "does not exist")

cmd/gexp/config.go

@@ -0,0 +1,186 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"unicode"
+
+	cli "gopkg.in/urfave/cli.v1"
+
+	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/contracts/release"
+	"github.com/expanse-org/go-expanse/eth"
+	"github.com/expanse-org/go-expanse/node"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/naoina/toml"
+)
+
+var (
+	dumpConfigCommand = cli.Command{
+		Action:      dumpConfig,
+		Name:        "dumpconfig",
+		Usage:       "Show configuration values",
+		ArgsUsage:   "",
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: `The dumpconfig command shows configuration values.`,
+	}
+
+	configFileFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "TOML configuration file",
+	}
+)
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}
+
+type ethstatsConfig struct {
+	URL string `toml:",omitempty"`
+}
+
+type gethConfig struct {
+	Eth      eth.Config
+	Node     node.Config
+	Ethstats ethstatsConfig
+}
+
+func loadConfig(file string, cfg *gethConfig) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(cfg)
+	// Add file name to errors that have a line number.
+	if _, ok := err.(*toml.LineError); ok {
+		err = errors.New(file + ", " + err.Error())
+	}
+	return err
+}
+
+func defaultNodeConfig() node.Config {
+	cfg := node.DefaultConfig
+	cfg.Name = clientIdentifier
+	cfg.Version = params.VersionWithCommit(gitCommit)
+	cfg.HTTPModules = append(cfg.HTTPModules, "eth, exp")
+	cfg.WSModules = append(cfg.WSModules, "eth, exp")
+	cfg.IPCPath = "gexp.ipc"
+	return cfg
+}
+
+func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
+	// Load defaults.
+	cfg := gethConfig{
+		Eth:  eth.DefaultConfig,
+		Node: defaultNodeConfig(),
+	}
+
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+
+	// Apply flags.
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	stack, err := node.New(&cfg.Node)
+	if err != nil {
+		utils.Fatalf("Failed to create the protocol stack: %v", err)
+	}
+	utils.SetEthConfig(ctx, stack, &cfg.Eth)
+	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
+		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
+	}
+
+	return stack, cfg
+}
+
+func makeFullNode(ctx *cli.Context) *node.Node {
+	stack, cfg := makeConfigNode(ctx)
+
+	utils.RegisterEthService(stack, &cfg.Eth)
+
+	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
+	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	if shhEnabled || shhAutoEnabled {
+		utils.RegisterShhService(stack)
+	}
+
+	// Add the Ethereum Stats daemon if requested.
+	if cfg.Ethstats.URL != "" {
+		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
+	}
+
+	// Add the release oracle service so it boots along with node.
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		config := release.Config{
+			Oracle: relOracle,
+			Major:  uint32(params.VersionMajor),
+			Minor:  uint32(params.VersionMinor),
+			Patch:  uint32(params.VersionPatch),
+		}
+		commit, _ := hex.DecodeString(gitCommit)
+		copy(config.Commit[:], commit)
+		return release.NewReleaseService(ctx, config)
+	}); err != nil {
+		utils.Fatalf("Failed to register the Gexp release oracle service: %v", err)
+	}
+	return stack
+}
+
+// dumpConfig is the dumpconfig command.
+func dumpConfig(ctx *cli.Context) error {
+	_, cfg := makeConfigNode(ctx)
+	comment := ""
+
+	if cfg.Eth.Genesis != nil {
+		cfg.Eth.Genesis = nil
+		comment += "# Note: this config doesn't contain the genesis block.\n\n"
+	}
+
+	out, err := tomlSettings.Marshal(&cfg)
+	if err != nil {
+		return err
+	}
+	io.WriteString(os.Stdout, comment)
+	os.Stdout.Write(out)
+	return nil
+}

cmd/gexp/consolecmd_test.go

@@ -22,14 +22,17 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/rpc"
+)
+
+const (
+	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
+	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
 // Tests that a node embedded within a console can be started up properly and
@@ -45,24 +48,21 @@ func TestConsoleWelcome(t *testing.T) {
 
 	// Gather all the infos the welcome message needs to contain
 	gexp.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	gexp.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	gexp.setTemplateFunc("gover", runtime.Version)
 	gexp.setTemplateFunc("gethver", func() string { return params.Version })
 	gexp.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
-	gexp.setTemplateFunc("apis", func() []string {
-		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		sort.Strings(apis)
-		return apis
-	})
+	gexp.setTemplateFunc("apis", func() string { return ipcAPIs })
 
 	// Verify the actual welcome message to the required template
 	gexp.expect(`
 Welcome to the Gexp JavaScript console!
 
-instance: Gexp/v{{gethver}}/{{goos}}/{{gover}}
+instance: Gexp/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{.Etherbase}}
 at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit"}}
 `)
@@ -88,7 +88,7 @@ func TestIPCAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "ipc:"+ipc)
+	testAttachWelcome(t, gexp, "ipc:"+ipc, ipcAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
@@ -102,7 +102,7 @@ func TestHTTPAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--rpc", "--rpcport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "http://localhost:"+port)
+	testAttachWelcome(t, gexp, "http://localhost:"+port, httpAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
@@ -117,13 +117,13 @@ func TestWSAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--ws", "--wsport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "ws://localhost:"+port)
+	testAttachWelcome(t, gexp, "ws://localhost:"+port, httpAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
 }
 
-func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint string) {
+func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint, apis string) {
 	// Attach to a running gexp note and terminate immediately
 	attach := runGeth(t, "attach", endpoint)
 	defer attach.expectExit()
@@ -131,32 +131,24 @@ func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint string) {
 
 	// Gather all the infos the welcome message needs to contain
 	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	attach.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	attach.setTemplateFunc("gover", runtime.Version)
 	attach.setTemplateFunc("gethver", func() string { return params.Version })
 	attach.setTemplateFunc("etherbase", func() string { return gexp.Etherbase })
 	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
 	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
 	attach.setTemplateFunc("datadir", func() string { return gexp.Datadir })
-	attach.setTemplateFunc("apis", func() []string {
-		var apis []string
-		if strings.HasPrefix(endpoint, "ipc") {
-			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		} else {
-			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
-		}
-		sort.Strings(apis)
-		return apis
-	})
+	attach.setTemplateFunc("apis", func() string { return apis })
 
 	// Verify the actual welcome message to the required template
 	attach.expect(`
 Welcome to the Gexp JavaScript console!
 
-instance: Gexp/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{etherbase}}
 at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit" }}
 `)

cmd/gexp/dao_test.go

@@ -84,27 +84,24 @@ var daoGenesisForkBlock = big.NewInt(314)
 // set in the database after various initialization procedures and invocations.
 func TestDAOForkBlockNewChain(t *testing.T) {
 	for i, arg := range []struct {
-		testnet     bool
 		genesis     string
 		expectBlock *big.Int
 		expectVote  bool
 	}{
 		// Test DAO Default Mainnet
-		{false, "", params.MainNetDAOForkBlock, true},
-		// test DAO Default Testnet
-		{true, "", params.TestNetDAOForkBlock, true},
+		{"", params.MainNetDAOForkBlock, true},
 		// test DAO Init Old Privnet
-		{false, daoOldGenesis, nil, false},
+		{daoOldGenesis, nil, false},
 		// test DAO Default No Fork Privnet
-		{false, daoNoForkGenesis, daoGenesisForkBlock, false},
+		{daoNoForkGenesis, daoGenesisForkBlock, false},
 		// test DAO Default Pro Fork Privnet
-		{false, daoProForkGenesis, daoGenesisForkBlock, true},
+		{daoProForkGenesis, daoGenesisForkBlock, true},
 	} {
-		testDAOForkBlockNewChain(t, i, arg.testnet, arg.genesis, arg.expectBlock, arg.expectVote)
+		testDAOForkBlockNewChain(t, i, arg.genesis, arg.expectBlock, arg.expectVote)
 	}
 }
 
-func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis string, expectBlock *big.Int, expectVote bool) {
+func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
 	datadir := tmpdir(t)
 	defer os.RemoveAll(datadir)
@@ -119,17 +116,11 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	} else {
 		// Force chain initialization
 		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
-		if testnet {
-			args = append(args, "--testnet")
-		}
 		gexp := runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...)
 		gexp.cmd.Wait()
 	}
 	// Retrieve the DAO config flag from the database
-	path := filepath.Join(datadir, "gexp", "chaindata")
-	if testnet && genesis == "" {
-		path = filepath.Join(datadir, "testnet", "gexp", "chaindata")
-	}
+	path := filepath.Join(datadir, "geth", "chaindata")
 	db, err := ethdb.NewLDBDatabase(path, 0, 0)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
@@ -137,9 +128,6 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	defer db.Close()
 
 	genesisHash := common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
-	if testnet {
-		genesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
-	}
 	if genesis != "" {
 		genesisHash = daoGenesisHash
 	}

cmd/gexp/main.go

@@ -18,7 +18,6 @@
 package main
 
 import (
-	"encoding/hex"
 	"fmt"
 	"os"
 	"runtime"
@@ -29,17 +28,13 @@ import (
 	"github.com/expanse-org/go-expanse/accounts/keystore"
 	"github.com/expanse-org/go-expanse/cmd/utils"
 	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/common/hexutil"
 	"github.com/expanse-org/go-expanse/console"
-	"github.com/expanse-org/go-expanse/contracts/release"
 	"github.com/expanse-org/go-expanse/eth"
 	"github.com/expanse-org/go-expanse/ethclient"
 	"github.com/expanse-org/go-expanse/internal/debug"
 	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/metrics"
 	"github.com/expanse-org/go-expanse/node"
-	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/rlp"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -82,6 +77,8 @@ func init() {
 		versionCommand,
 		bugCommand,
 		licenseCommand,
+		// See config.go
+		dumpConfigCommand,
 	}
 
 	app.Flags = []cli.Flag{
@@ -99,6 +96,7 @@ func init() {
 		utils.EthashDatasetsOnDiskFlag,
 		utils.FastSyncFlag,
 		utils.LightModeFlag,
+		utils.SyncModeFlag,
 		utils.LightServFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
@@ -129,16 +127,12 @@ func init() {
 		utils.WSApiFlag,
 		utils.WSAllowedOriginsFlag,
 		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
 		utils.IPCPathFlag,
 		utils.ExecFlag,
 		utils.PreloadJSFlag,
 		utils.WhisperEnabledFlag,
 		utils.DevModeFlag,
 		utils.TestNetFlag,
-		utils.VMForceJitFlag,
-		utils.VMJitCacheFlag,
-		utils.VMEnableJitFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.RPCCORSDomainFlag,
@@ -146,14 +140,10 @@ func init() {
 		utils.MetricsEnabledFlag,
 		utils.FakePoWFlag,
 		utils.NoCompactionFlag,
-		utils.SolcPathFlag,
-		utils.GpoMinGasPriceFlag,
-		utils.GpoMaxGasPriceFlag,
-		utils.GpoFullBlockRatioFlag,
-		utils.GpobaseStepDownFlag,
-		utils.GpobaseStepUpFlag,
-		utils.GpobaseCorrectionFactorFlag,
+		utils.GpoBlocksFlag,
+		utils.GpoPercentileFlag,
 		utils.ExtraDataFlag,
+		configFileFlag,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
 
@@ -165,12 +155,6 @@ func init() {
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		// This should be the only place where reporting is enabled
-		// because it is not intended to run while testing.
-		// In addition to this check, bad block reports are sent only
-		// for chains with the main network genesis block and network id 1.
-		eth.EnableBadBlockReporting = false
-
 		utils.SetupNetwork(ctx)
 		return nil
 	}
@@ -199,52 +183,6 @@ func gexp(ctx *cli.Context) error {
 	return nil
 }
 
-func makeFullNode(ctx *cli.Context) *node.Node {
-	// Create the default extradata and construct the base node
-	var clientInfo = struct {
-		Version   uint
-		Name      string
-		GoVersion string
-		Os        string
-	}{uint(params.VersionMajor<<16 | params.VersionMinor<<8 | params.VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
-	extra, err := rlp.EncodeToBytes(clientInfo)
-	if err != nil {
-		log.Warn("Failed to set canonical miner information", "err", err)
-	}
-	if uint64(len(extra)) > params.MaximumExtraDataSize {
-		log.Warn("Miner extra data exceed limit", "extra", hexutil.Bytes(extra), "limit", params.MaximumExtraDataSize)
-		extra = nil
-	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	utils.RegisterEthService(ctx, stack, extra)
-
-	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
-	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
-	if shhEnabled || shhAutoEnabled {
-		utils.RegisterShhService(stack)
-	}
-	// Add the Ethereum Stats daemon if requested
-	if url := ctx.GlobalString(utils.EthStatsURLFlag.Name); url != "" {
-		utils.RegisterEthStatsService(stack, url)
-	}
-	// Add the release oracle service so it boots along with node.
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		config := release.Config{
-			Oracle: relOracle,
-			Major:  uint32(params.VersionMajor),
-			Minor:  uint32(params.VersionMinor),
-			Patch:  uint32(params.VersionPatch),
-		}
-		commit, _ := hex.DecodeString(gitCommit)
-		copy(config.Commit[:], commit)
-		return release.NewReleaseService(ctx, config)
-	}); err != nil {
-		utils.Fatalf("Failed to register the Gexp release oracle service: %v", err)
-	}
-	return stack
-}
-
 // startNode boots up the system node and all registered protocols, after which
 // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the
 // miner.
@@ -303,7 +241,15 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("expanse service not running: %v", err)
 		}
-		if err := ethereum.StartMining(ctx.GlobalInt(utils.MinerThreadsFlag.Name)); err != nil {
+		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
+			type threaded interface {
+				SetThreads(threads int)
+			}
+			if th, ok := ethereum.Engine().(threaded); ok {
+				th.SetThreads(threads)
+			}
+		}
+		if err := ethereum.StartMining(true); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/gexp/misccmd.go

@@ -26,9 +26,9 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/eth"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -87,7 +87,7 @@ func makedag(ctx *cli.Context) error {
 				utils.Fatalf("Can't find dir")
 			}
 			fmt.Println("making DAG, this could take awhile...")
-			pow.MakeDataset(blockNum, dir)
+			ethash.MakeDataset(blockNum, dir)
 		}
 	default:
 		wrongArgs()
@@ -101,10 +101,11 @@ func version(ctx *cli.Context) error {
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
+	fmt.Println("Architecture:", runtime.GOARCH)
 	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
 	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
 	fmt.Println("Go Version:", runtime.Version())
-	fmt.Println("OS:", runtime.GOOS)
+	fmt.Println("Operating System:", runtime.GOOS)
 	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
 	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
 	return nil

cmd/gexp/usage.go

@@ -30,7 +30,7 @@ import (
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2016 The go-ethereum / go-expanse Authors
+   Copyright 2013-2017 The go-ethereum / go-expanse Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -64,14 +64,15 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "ETHEREUM",
 		Flags: []cli.Flag{
+			configFileFlag,
 			utils.DataDirFlag,
 			utils.KeyStoreDirFlag,
 			utils.NetworkIdFlag,
 			utils.TestNetFlag,
 			utils.DevModeFlag,
+			utils.SyncModeFlag,
+			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
-			utils.FastSyncFlag,
-			utils.LightModeFlag,
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
@@ -115,7 +116,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.WSApiFlag,
 			utils.WSAllowedOriginsFlag,
 			utils.IPCDisabledFlag,
-			utils.IPCApiFlag,
 			utils.IPCPathFlag,
 			utils.RPCCORSDomainFlag,
 			utils.JSpathFlag,
@@ -133,6 +133,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NATFlag,
 			utils.NoDiscoverFlag,
 			utils.DiscoveryV5Flag,
+			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
 		},
@@ -151,43 +152,37 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "GAS PRICE ORACLE",
 		Flags: []cli.Flag{
-			utils.GpoMinGasPriceFlag,
-			utils.GpoMaxGasPriceFlag,
-			utils.GpoFullBlockRatioFlag,
-			utils.GpobaseStepDownFlag,
-			utils.GpobaseStepUpFlag,
-			utils.GpobaseCorrectionFactorFlag,
+			utils.GpoBlocksFlag,
+			utils.GpoPercentileFlag,
 		},
 	},
 	{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
-			utils.VMEnableJitFlag,
-			utils.VMForceJitFlag,
-			utils.VMJitCacheFlag,
 			utils.VMEnableDebugFlag,
 		},
 	},
 	{
 		Name: "LOGGING AND DEBUGGING",
 		Flags: append([]cli.Flag{
-			utils.EthStatsURLFlag,
 			utils.MetricsEnabledFlag,
 			utils.FakePoWFlag,
+			utils.NoCompactionFlag,
 		}, debug.Flags...),
 	},
+	{
+		Name: "DEPRECATED",
+		Flags: []cli.Flag{
+			utils.FastSyncFlag,
+			utils.LightModeFlag,
+		},
+	},
 	{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
 		},
 	},
-	{
-		Name: "MISCELLANEOUS",
-		Flags: []cli.Flag{
-			utils.SolcPathFlag,
-		},
-	},
 }
 
 func init() {

cmd/puppeth/module.go

@@ -0,0 +1,152 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+var (
+	// ErrServiceUnknown is returned when a service container doesn't exist.
+	ErrServiceUnknown = errors.New("service unknown")
+
+	// ErrServiceOffline is returned when a service container exists, but it is not
+	// running.
+	ErrServiceOffline = errors.New("service offline")
+
+	// ErrServiceUnreachable is returned when a service container is running, but
+	// seems to not respond to communication attempts.
+	ErrServiceUnreachable = errors.New("service unreachable")
+
+	// ErrNotExposed is returned if a web-service doesn't have an exposed port, nor
+	// a reverse-proxy in front of it to forward requests.
+	ErrNotExposed = errors.New("service not exposed, nor proxied")
+)
+
+// containerInfos is a heavily reduced version of the huge inspection dataset
+// returned from docker inspect, parsed into a form easily usable by puppeth.
+type containerInfos struct {
+	running bool              // Flag whether the container is running currently
+	envvars map[string]string // Collection of environmental variables set on the container
+	portmap map[string]int    // Port mapping from internal port/proto combos to host binds
+	volumes map[string]string // Volume mount points from container to host directories
+}
+
+// inspectContainer runs docker inspect against a running container
+func inspectContainer(client *sshClient, container string) (*containerInfos, error) {
+	// Check whether there's a container running for the service
+	out, err := client.Run(fmt.Sprintf("docker inspect %s", container))
+	if err != nil {
+		return nil, ErrServiceUnknown
+	}
+	// If yes, extract various configuration options
+	type inspection struct {
+		State struct {
+			Running bool
+		}
+		Mounts []struct {
+			Source      string
+			Destination string
+		}
+		Config struct {
+			Env []string
+		}
+		HostConfig struct {
+			PortBindings map[string][]map[string]string
+		}
+	}
+	var inspects []inspection
+	if err = json.Unmarshal(out, &inspects); err != nil {
+		return nil, err
+	}
+	inspect := inspects[0]
+
+	// Infos retrieved, parse the above into something meaningful
+	infos := &containerInfos{
+		running: inspect.State.Running,
+		envvars: make(map[string]string),
+		portmap: make(map[string]int),
+		volumes: make(map[string]string),
+	}
+	for _, envvar := range inspect.Config.Env {
+		if parts := strings.Split(envvar, "="); len(parts) == 2 {
+			infos.envvars[parts[0]] = parts[1]
+		}
+	}
+	for portname, details := range inspect.HostConfig.PortBindings {
+		if len(details) > 0 {
+			port, _ := strconv.Atoi(details[0]["HostPort"])
+			infos.portmap[portname] = port
+		}
+	}
+	for _, mount := range inspect.Mounts {
+		infos.volumes[mount.Destination] = mount.Source
+	}
+	return infos, err
+}
+
+// tearDown connects to a remote machine via SSH and terminates docker containers
+// running with the specified name in the specified network.
+func tearDown(client *sshClient, network string, service string, purge bool) ([]byte, error) {
+	// Tear down the running (or paused) container
+	out, err := client.Run(fmt.Sprintf("docker rm -f %s_%s_1", network, service))
+	if err != nil {
+		return out, err
+	}
+	// If requested, purge the associated docker image too
+	if purge {
+		return client.Run(fmt.Sprintf("docker rmi %s/%s", network, service))
+	}
+	return nil, nil
+}
+
+// resolve retrieves the hostname a service is running on either by returning the
+// actual server name and port, or preferably an nginx virtual host if available.
+func resolve(client *sshClient, network string, service string, port int) (string, error) {
+	// Inspect the service to get various configurations from it
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, service))
+	if err != nil {
+		return "", err
+	}
+	if !infos.running {
+		return "", ErrServiceOffline
+	}
+	// Container online, extract any environmental variables
+	if vhost := infos.envvars["VIRTUAL_HOST"]; vhost != "" {
+		return vhost, nil
+	}
+	return fmt.Sprintf("%s:%d", client.server, port), nil
+}
+
+// checkPort tries to connect to a remote host on a given
+func checkPort(host string, port int) error {
+	log.Trace("Verifying remote TCP connectivity", "server", host, "port", port)
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), time.Second)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}

cmd/puppeth/module_ethstats.go

@@ -0,0 +1,159 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// ethstatsDockerfile is the Dockerfile required to build an ethstats backend
+// and associated monitoring site.
+var ethstatsDockerfile = `
+FROM mhart/alpine-node:latest
+
+RUN \
+  apk add --update git                                         && \
+  git clone --depth=1 https://github.com/karalabe/eth-netstats && \
+	apk del git && rm -rf /var/cache/apk/*                       && \
+	\
+  cd /eth-netstats && npm install && npm install -g grunt-cli && grunt
+
+WORKDIR /eth-netstats
+EXPOSE 3000
+
+RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: []};' > lib/utils/config.js
+
+CMD ["npm", "start"]
+`
+
+// ethstatsComposefile is the docker-compose.yml file required to deploy and
+// maintain an ethstats monitoring site.
+var ethstatsComposefile = `
+version: '2'
+services:
+  ethstats:
+    build: .
+    image: {{.Network}}/ethstats{{if not .VHost}}
+    ports:
+      - "{{.Port}}:3000"{{end}}
+    environment:
+      - WS_SECRET={{.Secret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}{{end}}
+    restart: always
+`
+
+// deployEthstats deploys a new ethstats container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	for i, address := range trusted {
+		trusted[i] = fmt.Sprintf("\"%s\"", address)
+	}
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"Trusted": strings.Join(trusted, ", "),
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+		"Secret":  secret,
+		"VHost":   vhost,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// ethstatsInfos is returned from an ethstats status check to allow reporting
+// various configuration parameters.
+type ethstatsInfos struct {
+	host   string
+	port   int
+	secret string
+	config string
+}
+
+// String implements the stringer interface.
+func (info *ethstatsInfos) String() string {
+	return fmt.Sprintf("host=%s, port=%d, secret=%s", info.host, info.port, info.secret)
+}
+
+// checkEthstats does a health-check against an ethstats server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkEthstats(client *sshClient, network string) (*ethstatsInfos, error) {
+	// Inspect a possible ethstats container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_ethstats_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["3000/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and configure the connection string
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	secret := infos.envvars["WS_SECRET"]
+	config := fmt.Sprintf("%s@%s", secret, host)
+	if port != 80 && port != 443 {
+		config += fmt.Sprintf(":%d", port)
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Ethstats service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &ethstatsInfos{
+		host:   host,
+		port:   port,
+		secret: secret,
+		config: config,
+	}, nil
+}

cmd/puppeth/module_faucet.go

@@ -0,0 +1,210 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// faucetDockerfile is the Dockerfile required to build an faucet container to
+// grant crypto tokens based on GitHub authentications.
+var faucetDockerfile = `
+FROM alpine:latest
+
+RUN mkdir /go
+ENV GOPATH /go
+
+RUN \
+  apk add --update git go make gcc musl-dev ca-certificates linux-headers                             && \
+	mkdir -p $GOPATH/src/github.com/ethereum                                                            && \
+	(cd $GOPATH/src/github.com/ethereum && git clone --depth=1 https://github.com/expanse-org/go-expanse) && \
+  go build -v github.com/expanse-org/go-expanse/cmd/faucet                                              && \
+  apk del git go make gcc musl-dev linux-headers                                                      && \
+  rm -rf $GOPATH && rm -rf /var/cache/apk/*
+
+ADD genesis.json /genesis.json
+ADD account.json /account.json
+ADD account.pass /account.pass
+
+EXPOSE 8080
+
+CMD [ \
+	"/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}",            \
+	"--ethport", "{{.EthPort}}", "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", \
+	"--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass"     \
+]`
+
+// faucetComposefile is the docker-compose.yml file required to deploy and maintain
+// a crypto faucet.
+var faucetComposefile = `
+version: '2'
+services:
+  faucet:
+    build: .
+    image: {{.Network}}/faucet
+    ports:
+      - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}}
+      - "{{.ApiPort}}:8080"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.faucet
+    environment:
+      - ETH_PORT={{.EthPort}}
+      - ETH_NAME={{.EthName}}
+      - FAUCET_AMOUNT={{.FaucetAmount}}
+      - FAUCET_MINUTES={{.FaucetMinutes}}
+      - GITHUB_USER={{.GitHubUser}}
+      - GITHUB_TOKEN={{.GitHubToken}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}
+      - VIRTUAL_PORT=8080{{end}}
+    restart: always
+`
+
+// deployFaucet deploys a new faucet container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployFaucet(client *sshClient, network string, bootnodes []string, config *faucetInfos) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID":     config.node.network,
+		"Bootnodes":     strings.Join(bootnodes, ","),
+		"Ethstats":      config.node.ethstats,
+		"EthPort":       config.node.portFull,
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"FaucetName":    strings.Title(network),
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetComposefile)).Execute(composefile, map[string]interface{}{
+		"Network":       network,
+		"Datadir":       config.node.datadir,
+		"VHost":         config.host,
+		"ApiPort":       config.port,
+		"EthPort":       config.node.portFull,
+		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.node.genesis)
+	files[filepath.Join(workdir, "account.json")] = []byte(config.node.keyJSON)
+	files[filepath.Join(workdir, "account.pass")] = []byte(config.node.keyPass)
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the faucet service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// faucetInfos is returned from an faucet status check to allow reporting various
+// configuration parameters.
+type faucetInfos struct {
+	node        *nodeInfos
+	host        string
+	port        int
+	amount      int
+	minutes     int
+	githubUser  string
+	githubToken string
+}
+
+// String implements the stringer interface.
+func (info *faucetInfos) String() string {
+	return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, github=%s, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.githubUser, info.node.ethstats)
+}
+
+// checkFaucet does a health-check against an faucet server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
+	// Inspect a possible faucet container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_faucet_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["8080/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and the config values
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	amount, _ := strconv.Atoi(infos.envvars["FAUCET_AMOUNT"])
+	minutes, _ := strconv.Atoi(infos.envvars["FAUCET_MINUTES"])
+
+	// Retrieve the funding account informations
+	var out []byte
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.json", network)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.pass", network)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Faucet service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &faucetInfos{
+		node: &nodeInfos{
+			datadir:  infos.volumes["/root/.faucet"],
+			portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			ethstats: infos.envvars["ETH_NAME"],
+			keyJSON:  keyJSON,
+			keyPass:  keyPass,
+		},
+		host:        host,
+		port:        port,
+		amount:      amount,
+		minutes:     minutes,
+		githubUser:  infos.envvars["GITHUB_USER"],
+		githubToken: infos.envvars["GITHUB_TOKEN"],
+	}, nil
+}

cmd/puppeth/module_nginx.go

@@ -0,0 +1,106 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// nginxDockerfile is theis the Dockerfile required to build an nginx reverse-
+// proxy.
+var nginxDockerfile = `FROM jwilder/nginx-proxy`
+
+// nginxComposefile is the docker-compose.yml file required to deploy and maintain
+// an nginx reverse-proxy. The proxy is responsible for exposing one or more HTTP
+// services running on a single host.
+var nginxComposefile = `
+version: '2'
+services:
+  nginx:
+    build: .
+    image: {{.Network}}/nginx
+    ports:
+      - "{{.Port}}:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    restart: always
+`
+
+// deployNginx deploys a new nginx reverse-proxy container to expose one or more
+// HTTP services running on a single host. If an instance with the specified
+// network name already exists there, it will be overwritten!
+func deployNginx(client *sshClient, network string, port int) ([]byte, error) {
+	log.Info("Deploying nginx reverse-proxy", "server", client.server, "port", port)
+
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxDockerfile)).Execute(dockerfile, nil)
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nginxInfos is returned from an nginx reverse-proxy status check to allow
+// reporting various configuration parameters.
+type nginxInfos struct {
+	port int
+}
+
+// String implements the stringer interface.
+func (info *nginxInfos) String() string {
+	return fmt.Sprintf("port=%d", info.port)
+}
+
+// checkNginx does a health-check against an nginx reverse-proxy to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkNginx(client *sshClient, network string) (*nginxInfos, error) {
+	// Inspect a possible nginx container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_nginx_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Container available, assemble and return the useful infos
+	return &nginxInfos{
+		port: infos.portmap["80/tcp"],
+	}, nil
+}

cmd/puppeth/module_node.go

@@ -0,0 +1,222 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// nodeDockerfile is the Dockerfile required to run an Ethereum node.
+var nodeDockerfile = `
+FROM ethereum/client-go:alpine-develop
+
+ADD genesis.json /genesis.json
+{{if .Unlock}}
+	ADD signer.json /signer.json
+	ADD signer.pass /signer.pass
+{{end}}
+RUN \
+  echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
+	echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}}' >> geth.sh
+
+ENTRYPOINT ["/bin/sh", "geth.sh"]
+`
+
+// nodeComposefile is the docker-compose.yml file required to deploy and maintain
+// an Ethereum node (bootnode or miner for now).
+var nodeComposefile = `
+version: '2'
+services:
+  {{.Type}}:
+    build: .
+    image: {{.Network}}/{{.Type}}
+    ports:
+      - "{{.FullPort}}:{{.FullPort}}"
+      - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
+      - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.ethereum
+    environment:
+      - FULL_PORT={{.FullPort}}/tcp
+      - LIGHT_PORT={{.LightPort}}/udp
+      - TOTAL_PEERS={{.TotalPeers}}
+      - LIGHT_PEERS={{.LightPeers}}
+      - STATS_NAME={{.Ethstats}}
+      - MINER_NAME={{.Etherbase}}
+    restart: always
+`
+
+// deployNode deploys a new Ethereum node container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployNode(client *sshClient, network string, bootnodes []string, config *nodeInfos) ([]byte, error) {
+	kind := "sealnode"
+	if config.keyJSON == "" && config.etherbase == "" {
+		kind = "bootnode"
+		bootnodes = make([]string, 0)
+	}
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	lightFlag := ""
+	if config.peersLight > 0 {
+		lightFlag = fmt.Sprintf("--lightpeers=%d --lightserv=50", config.peersLight)
+	}
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID": config.network,
+		"Port":      config.portFull,
+		"Peers":     config.peersTotal,
+		"LightFlag": lightFlag,
+		"Bootnodes": strings.Join(bootnodes, ","),
+		"Ethstats":  config.ethstats,
+		"Etherbase": config.etherbase,
+		"Unlock":    config.keyJSON != "",
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeComposefile)).Execute(composefile, map[string]interface{}{
+		"Type":       kind,
+		"Datadir":    config.datadir,
+		"Network":    network,
+		"FullPort":   config.portFull,
+		"TotalPeers": config.peersTotal,
+		"Light":      config.peersLight > 0,
+		"LightPort":  config.portFull + 1,
+		"LightPeers": config.peersLight,
+		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
+		"Etherbase":  config.etherbase,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	//genesisfile, _ := json.MarshalIndent(config.genesis, "", "  ")
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.genesis)
+
+	if config.keyJSON != "" {
+		files[filepath.Join(workdir, "signer.json")] = []byte(config.keyJSON)
+		files[filepath.Join(workdir, "signer.pass")] = []byte(config.keyPass)
+	}
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the bootnode service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nodeInfos is returned from a boot or seal node status check to allow reporting
+// various configuration parameters.
+type nodeInfos struct {
+	genesis    []byte
+	network    int64
+	datadir    string
+	ethstats   string
+	portFull   int
+	portLight  int
+	enodeFull  string
+	enodeLight string
+	peersTotal int
+	peersLight int
+	etherbase  string
+	keyJSON    string
+	keyPass    string
+}
+
+// String implements the stringer interface.
+func (info *nodeInfos) String() string {
+	discv5 := ""
+	if info.peersLight > 0 {
+		discv5 = fmt.Sprintf(", portv5=%d", info.portLight)
+	}
+	return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s", info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats)
+}
+
+// checkNode does a health-check against an boot or seal node server to verify
+// whether it's running, and if yes, whether it's responsive.
+func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) {
+	kind := "bootnode"
+	if !boot {
+		kind = "sealnode"
+	}
+	// Inspect a possible bootnode container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, kind))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve a few types from the environmental variables
+	totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
+	lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
+
+	// Container available, retrieve its node ID and its genesis json
+	var out []byte
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 /geth --exec admin.nodeInfo.id attach", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	genesis := bytes.TrimSpace(out)
+
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.json", network, kind)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.pass", network, kind)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the devp2p is reachable
+	port := infos.portmap[infos.envvars["FULL_PORT"]]
+	if err = checkPort(client.server, port); err != nil {
+		log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
+	}
+	// Assemble and return the useful infos
+	stats := &nodeInfos{
+		genesis:    genesis,
+		datadir:    infos.volumes["/root/.ethereum"],
+		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
+		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
+		peersTotal: totalPeers,
+		peersLight: lightPeers,
+		ethstats:   infos.envvars["STATS_NAME"],
+		etherbase:  infos.envvars["MINER_NAME"],
+		keyJSON:    keyJSON,
+		keyPass:    keyPass,
+	}
+	stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
+	if stats.portLight != 0 {
+		stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
+	}
+	return stats, nil
+}

cmd/puppeth/puppeth.go

@@ -0,0 +1,55 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// puppeth is a command to assemble and maintain private networks.
+package main
+
+import (
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/expanse-org/go-expanse/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// main is just a boring entry point to set up the CLI app.
+func main() {
+	app := cli.NewApp()
+	app.Name = "puppeth"
+	app.Usage = "assemble and maintain private Ethereum networks"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "network",
+			Usage: "name of the network to administer",
+		},
+		cli.IntFlag{
+			Name:  "loglevel",
+			Value: 4,
+			Usage: "log level to emit to the screen",
+		},
+	}
+	app.Action = func(c *cli.Context) error {
+		// Set up the logger to print everything and the random generator
+		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
+		rand.Seed(time.Now().UnixNano())
+
+		// Start the wizard and relinquish control
+		makeWizard(c.String("network")).run()
+		return nil
+	}
+	app.Run(os.Args)
+}

cmd/puppeth/ssh.go

@@ -0,0 +1,195 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/expanse-org/go-expanse/log"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// sshClient is a small wrapper around Go's SSH client with a few utility methods
+// implemented on top.
+type sshClient struct {
+	server  string // Server name or IP without port number
+	address string // IP address of the remote server
+	client  *ssh.Client
+	logger  log.Logger
+}
+
+// dial establishes an SSH connection to a remote node using the current user and
+// the user's configured private RSA key.
+func dial(server string) (*sshClient, error) {
+	// Figure out a label for the server and a logger
+	label := server
+	if strings.Contains(label, ":") {
+		label = label[:strings.Index(label, ":")]
+	}
+	logger := log.New("server", label)
+	logger.Debug("Attempting to establish SSH connection")
+
+	user, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+	// Configure the supported authentication methods (private key and password)
+	var auths []ssh.AuthMethod
+
+	path := filepath.Join(user.HomeDir, ".ssh", "id_rsa")
+	if buf, err := ioutil.ReadFile(path); err != nil {
+		log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
+	} else {
+		key, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			log.Warn("Bad SSH key, falling back to passwords", "path", path, "err", err)
+		} else {
+			auths = append(auths, ssh.PublicKeys(key))
+		}
+	}
+	auths = append(auths, ssh.PasswordCallback(func() (string, error) {
+		fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", user.Username, server)
+		blob, err := terminal.ReadPassword(int(syscall.Stdin))
+
+		fmt.Println()
+		return string(blob), err
+	}))
+	// Resolve the IP address of the remote server
+	addr, err := net.LookupHost(label)
+	if err != nil {
+		return nil, err
+	}
+	if len(addr) == 0 {
+		return nil, errors.New("no IPs associated with domain")
+	}
+	// Try to dial in to the remote server
+	logger.Trace("Dialing remote SSH server", "user", user.Username, "key", path)
+	if !strings.Contains(server, ":") {
+		server += ":22"
+	}
+	client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: user.Username, Auth: auths})
+	if err != nil {
+		return nil, err
+	}
+	// Connection established, return our utility wrapper
+	c := &sshClient{
+		server:  label,
+		address: addr[0],
+		client:  client,
+		logger:  logger,
+	}
+	if err := c.init(); err != nil {
+		client.Close()
+		return nil, err
+	}
+	return c, nil
+}
+
+// init runs some initialization commands on the remote server to ensure it's
+// capable of acting as puppeth target.
+func (client *sshClient) init() error {
+	client.logger.Debug("Verifying if docker is available")
+	if out, err := client.Run("docker version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker configured incorrectly: %s", out)
+	}
+	client.logger.Debug("Verifying if docker-compose is available")
+	if out, err := client.Run("docker-compose version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker-compose configured incorrectly: %s", out)
+	}
+	return nil
+}
+
+// Close terminates the connection to an SSH server.
+func (client *sshClient) Close() error {
+	return client.client.Close()
+}
+
+// Run executes a command on the remote server and returns the combined output
+// along with any error status.
+func (client *sshClient) Run(cmd string) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Execute the command and return any output
+	client.logger.Trace("Running command on remote server", "cmd", cmd)
+	return session.CombinedOutput(cmd)
+}
+
+// Stream executes a command on the remote server and streams all outputs into
+// the local stdout and stderr streams.
+func (client *sshClient) Stream(cmd string) error {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	session.Stdout = os.Stdout
+	session.Stderr = os.Stderr
+
+	// Execute the command and return any output
+	client.logger.Trace("Streaming command on remote server", "cmd", cmd)
+	return session.Run(cmd)
+}
+
+// Upload copied the set of files to a remote server via SCP, creating any non-
+// existing folder in te mean time.
+func (client *sshClient) Upload(files map[string][]byte) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Create a goroutine that streams the SCP content
+	go func() {
+		out, _ := session.StdinPipe()
+		defer out.Close()
+
+		for file, content := range files {
+			client.logger.Trace("Uploading file to server", "file", file, "bytes", len(content))
+
+			fmt.Fprintln(out, "D0755", 0, filepath.Dir(file))             // Ensure the folder exists
+			fmt.Fprintln(out, "C0644", len(content), filepath.Base(file)) // Create the actual file
+			out.Write(content)                                            // Stream the data content
+			fmt.Fprint(out, "\x00")                                       // Transfer end with \x00
+			fmt.Fprintln(out, "E")                                        // Leave directory (simpler)
+		}
+	}()
+	return session.CombinedOutput("/usr/bin/scp -v -tr ./")
+}

cmd/puppeth/wizard.go

@@ -0,0 +1,229 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// config contains all the configurations needed by puppeth that should be saved
+// between sessions.
+type config struct {
+	path      string        // File containing the configuration values
+	genesis   *core.Genesis // Genesis block to cache for node deploys
+	bootFull  []string      // Bootnodes to always connect to by full nodes
+	bootLight []string      // Bootnodes to always connect to by light nodes
+	ethstats  string        // Ethstats settings to cache for node deploys
+
+	Servers []string `json:"servers,omitempty"`
+}
+
+// flush dumps the contents of config to disk.
+func (c config) flush() {
+	os.MkdirAll(filepath.Dir(c.path), 0755)
+
+	sort.Strings(c.Servers)
+	out, _ := json.MarshalIndent(c, "", "  ")
+	if err := ioutil.WriteFile(c.path, out, 0644); err != nil {
+		log.Warn("Failed to save puppeth configs", "file", c.path, "err", err)
+	}
+}
+
+type wizard struct {
+	network string // Network name to manage
+	conf    config // Configurations from previous runs
+
+	servers  map[string]*sshClient // SSH connections to servers to administer
+	services map[string][]string   // Ethereum services known to be running on servers
+
+	in *bufio.Reader // Wrapper around stdin to allow reading user input
+}
+
+// read reads a single line from stdin, trimming if from spaces.
+func (w *wizard) read() string {
+	fmt.Printf("> ")
+	text, err := w.in.ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+	return strings.TrimSpace(text)
+}
+
+// readString reads a single line from stdin, trimming if from spaces, enforcing
+// non-emptyness.
+func (w *wizard) readString() string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+	}
+}
+
+// readDefaultString reads a single line from stdin, trimming if from spaces. If
+// an empty line is entered, the default value is returned.
+func (w *wizard) readDefaultString(def string) string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+		return def
+	}
+}
+
+// readInt reads a single line from stdin, trimming if from spaces, enforcing it
+// to parse into an integer.
+func (w *wizard) readInt() int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			continue
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readDefaultInt reads a single line from stdin, trimming if from spaces, enforcing
+// it to parse into an integer. If an empty line is entered, the default value is
+// returned.
+func (w *wizard) readDefaultInt(def int) int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readPassword reads a single line from stdin, trimming it from the trailing new
+// line and returns it. The input will not be echoed.
+func (w *wizard) readPassword() string {
+	for {
+		fmt.Printf("> ")
+		text, err := terminal.ReadPassword(int(syscall.Stdin))
+		if err != nil {
+			log.Crit("Failed to read password", "err", err)
+		}
+		fmt.Println()
+		return string(text)
+	}
+}
+
+// readAddress reads a single line from stdin, trimming if from spaces and converts
+// it to an Ethereum address.
+func (w *wizard) readAddress() *common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return nil
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		address := common.BigToAddress(bigaddr)
+		return &address
+	}
+}
+
+// readDefaultAddress reads a single line from stdin, trimming if from spaces and
+// converts it to an Ethereum address. If an empty line is entered, the default
+// value is returned.
+func (w *wizard) readDefaultAddress(def common.Address) common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		return common.BigToAddress(bigaddr)
+	}
+}
+
+// readJSON reads a raw JSON message and returns it.
+func (w *wizard) readJSON() string {
+	var blob json.RawMessage
+
+	for {
+		fmt.Printf("> ")
+		if err := json.NewDecoder(w.in).Decode(&blob); err != nil {
+			log.Error("Invalid JSON, please try again", "err", err)
+			continue
+		}
+		return string(blob)
+	}
+}

cmd/puppeth/wizard_dashboard.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployDashboard queries the user for various input on deploying a web-service
+// dashboard, after which is pushes the container.
+func (w *wizard) deployDashboard() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active dashboard configurations from the server
+	infos, err := checkDashboard(client, w.network)
+	if err != nil {
+		infos = &dashboardInfos{
+			port: 80,
+			host: client.server,
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the dashboard listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy the dashboard on
+	infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host)
+	if err != nil {
+		log.Error("Failed to decide on dashboard host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out which services are available
+	available := make(map[string][]string)
+	for server, services := range w.services {
+		for _, service := range services {
+			available[service] = append(available[service], server)
+		}
+	}
+	listing := make(map[string]string)
+	for _, service := range []string{"ethstats", "explorer", "wallet", "faucet"} {
+		// Gather all the locally hosted pages of this type
+		var pages []string
+		for _, server := range available[service] {
+			client := w.servers[server]
+			if client == nil {
+				continue
+			}
+			// If there's a service running on the machine, retrieve it's port number
+			var port int
+			switch service {
+			case "ethstats":
+				if infos, err := checkEthstats(client, w.network); err == nil {
+					port = infos.port
+				}
+			case "faucet":
+				if infos, err := checkFaucet(client, w.network); err == nil {
+					port = infos.port
+				}
+			}
+			if page, err := resolve(client, w.network, service, port); err == nil && page != "" {
+				pages = append(pages, page)
+			}
+		}
+		// Promt the user to chose one, enter manually or simply not list this service
+		defLabel, defChoice := "don't list", len(pages)+2
+		if len(pages) > 0 {
+			defLabel, defChoice = pages[0], 1
+		}
+		fmt.Println()
+		fmt.Printf("Which %s service to list? (default = %s)\n", service, defLabel)
+		for i, page := range pages {
+			fmt.Printf(" %d. %s\n", i+1, page)
+		}
+		fmt.Printf(" %d. List external %s service\n", len(pages)+1, service)
+		fmt.Printf(" %d. Don't list any %s service\n", len(pages)+2, service)
+
+		choice := w.readDefaultInt(defChoice)
+		if choice < 0 || choice > len(pages)+2 {
+			log.Error("Invalid listing choice, aborting")
+			return
+		}
+		switch {
+		case choice <= len(pages):
+			listing[service] = pages[choice-1]
+		case choice == len(pages)+1:
+			fmt.Println()
+			fmt.Printf("Which address is the external %s service at?\n", service)
+			listing[service] = w.readString()
+		default:
+			// No service hosting for this
+		}
+	}
+	// If we have ethstats running, ask whether to make the secret public or not
+	var ethstats bool
+	if w.conf.ethstats != "" {
+		fmt.Println()
+		fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)")
+		ethstats = w.readDefaultString("y") == "y"
+	}
+	// Try to deploy the dashboard container on the host
+	if out, err := deployDashboard(client, w.network, infos.port, infos.host, listing, &w.conf, ethstats); err != nil {
+		log.Error("Failed to deploy dashboard container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_ethstats.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployEthstats queries the user for various input on deploying an ethstats
+// monitoring server, after which it executes it.
+func (w *wizard) deployEthstats() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkEthstats(client, w.network)
+	if err != nil {
+		infos = &ethstatsInfos{
+			port:   80,
+			host:   client.server,
+			secret: "",
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should ethstats listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on ethstats host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the secret and boot ethstats
+	fmt.Println()
+	if infos.secret == "" {
+		fmt.Printf("What should be the secret password for the API? (must not be empty)\n")
+		infos.secret = w.readString()
+	} else {
+		fmt.Printf("What should be the secret password for the API? (default = %s)\n", infos.secret)
+		infos.secret = w.readDefaultString(infos.secret)
+	}
+	// Try to deploy the ethstats server on the host
+	trusted := make([]string, 0, len(w.servers))
+	for _, client := range w.servers {
+		if client != nil {
+			trusted = append(trusted, client.address)
+		}
+	}
+	if out, err := deployEthstats(client, w.network, infos.port, infos.secret, infos.host, trusted); err != nil {
+		log.Error("Failed to deploy ethstats container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_faucet.go

@@ -0,0 +1,172 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployFaucet queries the user for various input on deploying a faucet, after
+// which it executes it.
+func (w *wizard) deployFaucet() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active faucet configurations from the server
+	infos, err := checkFaucet(client, w.network)
+	if err != nil {
+		infos = &faucetInfos{
+			node:    &nodeInfos{portFull: 30303, peersTotal: 25},
+			port:    80,
+			host:    client.server,
+			amount:  1,
+			minutes: 1440,
+		}
+	}
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.node.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the faucet listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on faucet host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the funcing amount per perdion configurations
+	fmt.Println()
+	fmt.Printf("How many Ethers to release per request? (default = %d)\n", infos.amount)
+	infos.amount = w.readDefaultInt(infos.amount)
+
+	fmt.Println()
+	fmt.Printf("How many minutes to enforce between requests? (default = %d)\n", infos.minutes)
+	infos.minutes = w.readDefaultInt(infos.minutes)
+
+	// Accessing GitHub gists requires API authorization, retrieve it
+	if infos.githubUser != "" {
+		fmt.Println()
+		fmt.Printf("Reused previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser)
+		if w.readDefaultString("y") != "y" {
+			infos.githubUser, infos.githubToken = "", ""
+		}
+	}
+	if infos.githubUser == "" {
+		// No previous authorization (or new one requested)
+		fmt.Println()
+		fmt.Println("Which GitHub user to verify Gists through?")
+		infos.githubUser = w.readString()
+
+		fmt.Println()
+		fmt.Println("What is the GitHub personal access token of the user? (won't be echoed)")
+		infos.githubToken = w.readPassword()
+
+		// Do a sanity check query against github to ensure it's valid
+		req, _ := http.NewRequest("GET", "https://api.github.com/user", nil)
+		req.SetBasicAuth(infos.githubUser, infos.githubToken)
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Error("Failed to verify GitHub authentication", "err", err)
+			return
+		}
+		defer res.Body.Close()
+
+		var msg struct {
+			Login   string `json:"login"`
+			Message string `json:"message"`
+		}
+		if err = json.NewDecoder(res.Body).Decode(&msg); err != nil {
+			log.Error("Failed to decode authorization response", "err", err)
+			return
+		}
+		if msg.Login != infos.githubUser {
+			log.Error("GitHub authorization failed", "user", infos.githubUser, "message", msg.Message)
+			return
+		}
+	}
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.node.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.node.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.node.datadir)
+		infos.node.datadir = w.readDefaultString(infos.node.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
+	infos.node.portFull = w.readDefaultInt(infos.node.portFull)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.node.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.node.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.node.ethstats)
+		infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats
+	}
+	// Load up the credential needed to release funds
+	if infos.node.keyJSON != "" {
+		var key keystore.Key
+		if err := json.Unmarshal([]byte(infos.node.keyJSON), &key); err != nil {
+			infos.node.keyJSON, infos.node.keyPass = "", ""
+		} else {
+			fmt.Println()
+			fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex())
+			if w.readDefaultString("y") != "y" {
+				infos.node.keyJSON, infos.node.keyPass = "", ""
+			}
+		}
+	}
+	if infos.node.keyJSON == "" {
+		fmt.Println()
+		fmt.Println("Please paste the faucet's funding account key JSON:")
+		infos.node.keyJSON = w.readJSON()
+
+		fmt.Println()
+		fmt.Println("What's the unlock password for the account? (won't be echoed)")
+		infos.node.keyPass = w.readPassword()
+
+		if _, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil {
+			log.Error("Failed to decrypt key with given passphrase")
+			return
+		}
+	}
+	// Try to deploy the faucet server on the host
+	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos); err != nil {
+		log.Error("Failed to deploy faucet container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_genesis.go

@@ -0,0 +1,136 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"time"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+// makeGenesis creates a new genesis struct based on some user input.
+func (w *wizard) makeGenesis() {
+	// Construct a default genesis block
+	genesis := &core.Genesis{
+		Timestamp:  uint64(time.Now().Unix()),
+		GasLimit:   4700000,
+		Difficulty: big.NewInt(1048576),
+		Alloc:      make(core.GenesisAlloc),
+		Config: &params.ChainConfig{
+			HomesteadBlock: big.NewInt(1),
+			EIP150Block:    big.NewInt(2),
+			EIP155Block:    big.NewInt(3),
+			EIP158Block:    big.NewInt(3),
+		},
+	}
+	// Figure out which consensus engine to choose
+	fmt.Println()
+	fmt.Println("Which consensus engine to use? (default = clique)")
+	fmt.Println(" 1. Ethash - proof-of-work")
+	fmt.Println(" 2. Clique - proof-of-authority")
+
+	choice := w.read()
+	switch {
+	case choice == "1":
+		// In case of ethash, we're pretty much done
+		genesis.Config.Ethash = new(params.EthashConfig)
+		genesis.ExtraData = make([]byte, 32)
+
+	case choice == "" || choice == "2":
+		// In the case of clique, configure the consensus parameters
+		genesis.Difficulty = big.NewInt(1)
+		genesis.Config.Clique = &params.CliqueConfig{
+			Period: 15,
+			Epoch:  30000,
+		}
+		fmt.Println()
+		fmt.Println("How many seconds should blocks take? (default = 15)")
+		genesis.Config.Clique.Period = uint64(w.readDefaultInt(15))
+
+		// We also need the initial list of signers
+		fmt.Println()
+		fmt.Println("Which accounts are allowed to seal? (mandatory at least one)")
+
+		var signers []common.Address
+		for {
+			if address := w.readAddress(); address != nil {
+				signers = append(signers, *address)
+				continue
+			}
+			if len(signers) > 0 {
+				break
+			}
+		}
+		// Sort the signers and embed into the extra-data section
+		for i := 0; i < len(signers); i++ {
+			for j := i + 1; j < len(signers); j++ {
+				if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
+					signers[i], signers[j] = signers[j], signers[i]
+				}
+			}
+		}
+		genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65)
+		for i, signer := range signers {
+			copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:])
+		}
+
+	default:
+		log.Crit("Invalid consensus engine choice", "choice", choice)
+	}
+	// Consensus all set, just ask for initial funds and go
+	fmt.Println()
+	fmt.Println("Which accounts should be pre-funded? (advisable at least one)")
+	for {
+		// Read the address of the account to fund
+		if address := w.readAddress(); address != nil {
+			genesis.Alloc[*address] = core.GenesisAccount{
+				Balance: new(big.Int).Lsh(big.NewInt(1), 256-7), // 2^256 / 128 (allow many pre-funds without balance overflows)
+			}
+			continue
+		}
+		break
+	}
+	// Add a batch of precompile balances to avoid them getting deleted
+	for i := int64(0); i < 256; i++ {
+		genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
+	}
+	fmt.Println()
+
+	// Query the user for some custom extras
+	fmt.Println()
+	fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)")
+	genesis.Config.ChainId = big.NewInt(int64(w.readDefaultInt(rand.Intn(65536))))
+
+	fmt.Println()
+	fmt.Println("Anything fun to embed into the genesis block? (max 32 bytes)")
+
+	extra := w.read()
+	if len(extra) > 32 {
+		extra = extra[:32]
+	}
+	genesis.ExtraData = append([]byte(extra), genesis.ExtraData[len(extra):]...)
+
+	// All done, store the genesis and flush to disk
+	w.conf.genesis = genesis
+}

cmd/puppeth/wizard_intro.go

@@ -0,0 +1,153 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// makeWizard creates and returns a new puppeth wizard.
+func makeWizard(network string) *wizard {
+	return &wizard{
+		network:  network,
+		servers:  make(map[string]*sshClient),
+		services: make(map[string][]string),
+		in:       bufio.NewReader(os.Stdin),
+	}
+}
+
+// run displays some useful infos to the user, starting on the journey of
+// setting up a new or managing an existing Ethereum private network.
+func (w *wizard) run() {
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println("| Welcome to puppeth, your Ethereum private network manager |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| This tool lets you create a new Ethereum network down to  |")
+	fmt.Println("| the genesis block, bootnodes, miners and ethstats servers |")
+	fmt.Println("| without the hassle that it would normally entail.         |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| Puppeth uses SSH to dial in to remote servers, and builds |")
+	fmt.Println("| its network components out of Docker containers using the |")
+	fmt.Println("| docker-compose toolset.                                   |")
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println()
+
+	// Make sure we have a good network name to work with	fmt.Println()
+	if w.network == "" {
+		fmt.Println("Please specify a network name to administer (no spaces, please)")
+		for {
+			w.network = w.readString()
+			if !strings.Contains(w.network, " ") {
+				fmt.Printf("Sweet, you can set this via --network=%s next time!\n\n", w.network)
+				break
+			}
+			log.Error("I also like to live dangerously, still no spaces")
+		}
+	}
+	log.Info("Administering Ethereum network", "name", w.network)
+
+	// Load initial configurations and connect to all live servers
+	w.conf.path = filepath.Join(os.Getenv("HOME"), ".puppeth", w.network)
+
+	blob, err := ioutil.ReadFile(w.conf.path)
+	if err != nil {
+		log.Warn("No previous configurations found", "path", w.conf.path)
+	} else if err := json.Unmarshal(blob, &w.conf); err != nil {
+		log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err)
+	} else {
+		for _, server := range w.conf.Servers {
+			log.Info("Dialing previously configured server", "server", server)
+			client, err := dial(server)
+			if err != nil {
+				log.Error("Previous server unreachable", "server", server, "err", err)
+			}
+			w.servers[server] = client
+		}
+		w.networkStats(false)
+	}
+	// Basics done, loop ad infinitum about what to do
+	for {
+		fmt.Println()
+		fmt.Println("What would you like to do? (default = stats)")
+		fmt.Println(" 1. Show network stats")
+		if w.conf.genesis == nil {
+			fmt.Println(" 2. Configure new genesis")
+		} else {
+			fmt.Println(" 2. Save existing genesis")
+		}
+		if len(w.servers) == 0 {
+			fmt.Println(" 3. Track new remote server")
+		} else {
+			fmt.Println(" 3. Manage tracked machines")
+		}
+		if len(w.services) == 0 {
+			fmt.Println(" 4. Deploy network components")
+		} else {
+			fmt.Println(" 4. Manage network components")
+		}
+		//fmt.Println(" 5. ProTips for common usecases")
+
+		choice := w.read()
+		switch {
+		case choice == "" || choice == "1":
+			w.networkStats(false)
+
+		case choice == "2":
+			// If we don't have a genesis, make one
+			if w.conf.genesis == nil {
+				w.makeGenesis()
+			} else {
+				// Otherwise just save whatever we currently have
+				fmt.Println()
+				fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
+				out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
+				if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
+					log.Error("Failed to save genesis file", "err", err)
+				}
+				log.Info("Exported existing genesis block")
+			}
+		case choice == "3":
+			if len(w.servers) == 0 {
+				if w.makeServer() != "" {
+					w.networkStats(false)
+				}
+			} else {
+				w.manageServers()
+			}
+		case choice == "4":
+			if len(w.services) == 0 {
+				w.deployComponent()
+			} else {
+				w.manageComponents()
+			}
+
+		case choice == "5":
+			w.networkStats(true)
+
+		default:
+			log.Error("That's not something I can do")
+		}
+	}
+}

cmd/puppeth/wizard_netstats.go

@@ -0,0 +1,235 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/olekukonko/tablewriter"
+)
+
+// networkStats verifies the status of network components and generates a protip
+// configuration set to give users hints on how to do various tasks.
+func (w *wizard) networkStats(tips bool) {
+	if len(w.servers) == 0 {
+		log.Error("No remote machines to gather stats from")
+		return
+	}
+	protips := new(protips)
+
+	// Iterate over all the specified hosts and check their status
+	stats := tablewriter.NewWriter(os.Stdout)
+	stats.SetHeader([]string{"Server", "IP", "Status", "Service", "Details"})
+	stats.SetColWidth(128)
+
+	for _, server := range w.conf.Servers {
+		client := w.servers[server]
+		logger := log.New("server", server)
+		logger.Info("Starting remote server health-check")
+
+		// If the server is not connected, try to connect again
+		if client == nil {
+			conn, err := dial(server)
+			if err != nil {
+				logger.Error("Failed to establish remote connection", "err", err)
+				stats.Append([]string{server, "", err.Error(), "", ""})
+				continue
+			}
+			client = conn
+		}
+		// Client connected one way or another, run health-checks
+		services := make(map[string]string)
+		logger.Debug("Checking for nginx availability")
+		if infos, err := checkNginx(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["nginx"] = err.Error()
+			}
+		} else {
+			services["nginx"] = infos.String()
+		}
+		logger.Debug("Checking for ethstats availability")
+		if infos, err := checkEthstats(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["ethstats"] = err.Error()
+			}
+		} else {
+			services["ethstats"] = infos.String()
+			protips.ethstats = infos.config
+		}
+		logger.Debug("Checking for bootnode availability")
+		if infos, err := checkNode(client, w.network, true); err != nil {
+			if err != ErrServiceUnknown {
+				services["bootnode"] = err.Error()
+			}
+		} else {
+			services["bootnode"] = infos.String()
+
+			protips.genesis = string(infos.genesis)
+			protips.bootFull = append(protips.bootFull, infos.enodeFull)
+			if infos.enodeLight != "" {
+				protips.bootLight = append(protips.bootLight, infos.enodeLight)
+			}
+		}
+		logger.Debug("Checking for sealnode availability")
+		if infos, err := checkNode(client, w.network, false); err != nil {
+			if err != ErrServiceUnknown {
+				services["sealnode"] = err.Error()
+			}
+		} else {
+			services["sealnode"] = infos.String()
+			protips.genesis = string(infos.genesis)
+		}
+		logger.Debug("Checking for faucet availability")
+		if infos, err := checkFaucet(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["faucet"] = err.Error()
+			}
+		} else {
+			services["faucet"] = infos.String()
+		}
+		logger.Debug("Checking for dashboard availability")
+		if infos, err := checkDashboard(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["dashboard"] = err.Error()
+			}
+		} else {
+			services["dashboard"] = infos.String()
+		}
+		// All status checks complete, report and check next server
+		delete(w.services, server)
+		for service := range services {
+			w.services[server] = append(w.services[server], service)
+		}
+		server, address := client.server, client.address
+		for service, status := range services {
+			stats.Append([]string{server, address, "online", service, status})
+			server, address = "", ""
+		}
+		if len(services) == 0 {
+			stats.Append([]string{server, address, "online", "", ""})
+		}
+	}
+	// If a genesis block was found, load it into our configs
+	if protips.genesis != "" {
+		genesis := new(core.Genesis)
+		if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
+			log.Error("Failed to parse remote genesis", "err", err)
+		} else {
+			w.conf.genesis = genesis
+			protips.network = genesis.Config.ChainId.Int64()
+		}
+	}
+	if protips.ethstats != "" {
+		w.conf.ethstats = protips.ethstats
+	}
+	w.conf.bootFull = protips.bootFull
+	w.conf.bootLight = protips.bootLight
+
+	// Print any collected stats and return
+	if !tips {
+		stats.Render()
+	} else {
+		protips.print(w.network)
+	}
+}
+
+// protips contains a collection of network infos to report pro-tips
+// based on.
+type protips struct {
+	genesis   string
+	network   int64
+	bootFull  []string
+	bootLight []string
+	ethstats  string
+}
+
+// print analyzes the network information available and prints a collection of
+// pro tips for the user's consideration.
+func (p *protips) print(network string) {
+	// If a known genesis block is available, display it and prepend an init command
+	fullinit, lightinit := "", ""
+	if p.genesis != "" {
+		fullinit = fmt.Sprintf("geth --datadir=$HOME/.%s init %s.json && ", network, network)
+		lightinit = fmt.Sprintf("geth --datadir=$HOME/.%s --light init %s.json && ", network, network)
+	}
+	// If an ethstats server is available, add the ethstats flag
+	statsflag := ""
+	if p.ethstats != "" {
+		if strings.Contains(p.ethstats, " ") {
+			statsflag = fmt.Sprintf(` --ethstats="yournode:%s"`, p.ethstats)
+		} else {
+			statsflag = fmt.Sprintf(` --ethstats=yournode:%s`, p.ethstats)
+		}
+	}
+	// If bootnodes have been specified, add the bootnode flag
+	bootflagFull := ""
+	if len(p.bootFull) > 0 {
+		bootflagFull = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootFull, ","))
+	}
+	bootflagLight := ""
+	if len(p.bootLight) > 0 {
+		bootflagLight = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootLight, ","))
+	}
+	// Assemble all the known pro-tips
+	var tasks, tips []string
+
+	tasks = append(tasks, "Run an archive node with historical data")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=1024%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a full node with recent data only")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=512 --fast%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a light node with on demand retrievals")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	tasks = append(tasks, "Run an embedded node with constrained memory")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=32 --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	// If the tips are short, display in a table
+	short := true
+	for _, tip := range tips {
+		if len(tip) > 100 {
+			short = false
+			break
+		}
+	}
+	fmt.Println()
+	if short {
+		howto := tablewriter.NewWriter(os.Stdout)
+		howto.SetHeader([]string{"Fun tasks for you", "Tips on how to"})
+		howto.SetColWidth(100)
+
+		for i := 0; i < len(tasks); i++ {
+			howto.Append([]string{tasks[i], tips[i]})
+		}
+		howto.Render()
+		return
+	}
+	// Meh, tips got ugly, split into many lines
+	for i := 0; i < len(tasks); i++ {
+		fmt.Println(tasks[i])
+		fmt.Println(strings.Repeat("-", len(tasks[i])))
+		fmt.Println(tips[i])
+		fmt.Println()
+		fmt.Println()
+	}
+}

cmd/puppeth/wizard_network.go

@@ -0,0 +1,194 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// manageServers displays a list of servers the user can disconnect from, and an
+// option to connect to new servers.
+func (w *wizard) manageServers() {
+	// List all the servers we can disconnect, along with an entry to connect a new one
+	fmt.Println()
+	for i, server := range w.conf.Servers {
+		fmt.Printf(" %d. Disconnect %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return
+	}
+	// If the user selected an existing server, drop it
+	if choice <= len(w.conf.Servers) {
+		server := w.conf.Servers[choice-1]
+		client := w.servers[server]
+
+		delete(w.servers, server)
+		if client != nil {
+			client.Close()
+		}
+		w.conf.Servers = append(w.conf.Servers[:choice-1], w.conf.Servers[choice:]...)
+		w.conf.flush()
+
+		log.Info("Disconnected existing server", "server", server)
+		w.networkStats(false)
+		return
+	}
+	// If the user requested connecting a new server, do it
+	if w.makeServer() != "" {
+		w.networkStats(false)
+	}
+}
+
+// makeServer reads a single line from stdin and interprets it as a hostname to
+// connect to. It tries to establish a new SSH session and also executing some
+// baseline validations.
+//
+// If connection succeeds, the server is added to the wizards configs!
+func (w *wizard) makeServer() string {
+	fmt.Println()
+	fmt.Println("Please enter remote server's address:")
+
+	for {
+		// Read and fial the server to ensure docker is present
+		input := w.readString()
+
+		client, err := dial(input)
+		if err != nil {
+			log.Error("Server not ready for puppeth", "err", err)
+			return ""
+		}
+		// All checks passed, start tracking the server
+		w.servers[input] = client
+		w.conf.Servers = append(w.conf.Servers, input)
+		w.conf.flush()
+
+		return input
+	}
+}
+
+// selectServer lists the user all the currnetly known servers to choose from,
+// also granting the option to add a new one.
+func (w *wizard) selectServer() string {
+	// List the available server to the user and wait for a choice
+	fmt.Println()
+	fmt.Println("Which server do you want to interact with?")
+	for i, server := range w.conf.Servers {
+		fmt.Printf(" %d. %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return ""
+	}
+	// If the user requested connecting to a new server, go for it
+	if choice <= len(w.conf.Servers) {
+		return w.conf.Servers[choice-1]
+	}
+	return w.makeServer()
+}
+
+// manageComponents displays a list of network components the user can tear down
+// and an option
+func (w *wizard) manageComponents() {
+	// List all the componens we can tear down, along with an entry to deploy a new one
+	fmt.Println()
+
+	var serviceHosts, serviceNames []string
+	for server, services := range w.services {
+		for _, service := range services {
+			serviceHosts = append(serviceHosts, server)
+			serviceNames = append(serviceNames, service)
+
+			fmt.Printf(" %d. Tear down %s on %s\n", len(serviceHosts), strings.Title(service), server)
+		}
+	}
+	fmt.Printf(" %d. Deploy new network component\n", len(serviceHosts)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(serviceHosts)+1 {
+		log.Error("Invalid component choice, aborting")
+		return
+	}
+	// If the user selected an existing service, destroy it
+	if choice <= len(serviceHosts) {
+		// Figure out the service to destroy and execute it
+		service := serviceNames[choice-1]
+		server := serviceHosts[choice-1]
+		client := w.servers[server]
+
+		if out, err := tearDown(client, w.network, service, true); err != nil {
+			log.Error("Failed to tear down component", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return
+		}
+		// Clean up any references to it from out state
+		services := w.services[server]
+		for i, name := range services {
+			if name == service {
+				w.services[server] = append(services[:i], services[i+1:]...)
+				if len(w.services[server]) == 0 {
+					delete(w.services, server)
+				}
+			}
+		}
+		log.Info("Torn down existing component", "server", server, "service", service)
+		return
+	}
+	// If the user requested deploying a new component, do it
+	w.deployComponent()
+}
+
+// deployComponent displays a list of network components the user can deploy and
+// guides through the process.
+func (w *wizard) deployComponent() {
+	// Print all the things we can deploy and wait or user choice
+	fmt.Println()
+	fmt.Println("What would you like to deploy? (recommended order)")
+	fmt.Println(" 1. Ethstats  - Network monitoring tool")
+	fmt.Println(" 2. Bootnode  - Entry point of the network")
+	fmt.Println(" 3. Sealer    - Full node minting new blocks")
+	fmt.Println(" 4. Wallet    - Browser wallet for quick sends (todo)")
+	fmt.Println(" 5. Faucet    - Crypto faucet to give away funds")
+	fmt.Println(" 6. Dashboard - Website listing above web-services")
+
+	switch w.read() {
+	case "1":
+		w.deployEthstats()
+	case "2":
+		w.deployNode(true)
+	case "3":
+		w.deployNode(false)
+	case "4":
+	case "5":
+		w.deployFaucet()
+	case "6":
+		w.deployDashboard()
+	default:
+		log.Error("That's not something I can do")
+	}
+}

cmd/puppeth/wizard_nginx.go

@@ -0,0 +1,58 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// ensureVirtualHost checks whether a reverse-proxy is running on the specified
+// host machine, and if yes requests a virtual host from the user to host a
+// specific web service on. If no proxy exists, the method will offer to deploy
+// one.
+//
+// If the user elects not to use a reverse proxy, an empty hostname is returned!
+func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (string, error) {
+	if proxy, _ := checkNginx(client, w.network); proxy != nil {
+		// Reverse proxy is running, if ports match, we need a virtual host
+		if proxy.port == port {
+			fmt.Println()
+			fmt.Printf("Shared port, which domain to assign? (default = %s)\n", def)
+			return w.readDefaultString(def), nil
+		}
+	}
+	// Reverse proxy is not running, offer to deploy a new one
+	fmt.Println()
+	fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)")
+	if w.readDefaultString("y") == "y" {
+		if out, err := deployNginx(client, w.network, port); err != nil {
+			log.Error("Failed to deploy reverse-proxy", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return "", err
+		}
+		// Reverse proxy deployed, ask again for the virtual-host
+		fmt.Println()
+		fmt.Printf("Proxy deployed, which domain to assign? (default = %s)\n", def)
+		return w.readDefaultString(def), nil
+	}
+	// Reverse proxy not requested, deploy as a standalone service
+	return "", nil
+}

cmd/puppeth/wizard_node.go

@@ -0,0 +1,153 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployNode creates a new node configuration based on some user input.
+func (w *wizard) deployNode(boot bool) {
+	// Do some sanity check before the user wastes time on input
+	if w.conf.genesis == nil {
+		log.Error("No genesis block configured")
+		return
+	}
+	if w.conf.ethstats == "" {
+		log.Error("No ethstats server configured")
+		return
+	}
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkNode(client, w.network, boot)
+	if err != nil {
+		if boot {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
+		} else {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0}
+		}
+	}
+	infos.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		infos.datadir = w.readDefaultString(infos.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
+	infos.portFull = w.readDefaultInt(infos.portFull)
+
+	// Figure out how many peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many peers to allow connecting? (default = %d)\n", infos.peersTotal)
+	infos.peersTotal = w.readDefaultInt(infos.peersTotal)
+
+	// Figure out how many light peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many light peers to allow connecting? (default = %d)\n", infos.peersLight)
+	infos.peersLight = w.readDefaultInt(infos.peersLight)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.ethstats)
+		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+	}
+	// If the node is a miner/signer, load up needed credentials
+	if !boot {
+		if w.conf.genesis.Config.Ethash != nil {
+			// Ethash based miners only need an etherbase to mine against
+			fmt.Println()
+			if infos.etherbase == "" {
+				fmt.Printf("What address should the miner user?\n")
+				for {
+					if address := w.readAddress(); address != nil {
+						infos.etherbase = address.Hex()
+						break
+					}
+				}
+			} else {
+				fmt.Printf("What address should the miner user? (default = %s)\n", infos.etherbase)
+				infos.etherbase = w.readDefaultAddress(common.HexToAddress(infos.etherbase)).Hex()
+			}
+		} else if w.conf.genesis.Config.Clique != nil {
+			// If a previous signer was already set, offer to reuse it
+			if infos.keyJSON != "" {
+				var key keystore.Key
+				if err := json.Unmarshal([]byte(infos.keyJSON), &key); err != nil {
+					infos.keyJSON, infos.keyPass = "", ""
+				} else {
+					fmt.Println()
+					fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex())
+					if w.readDefaultString("y") != "y" {
+						infos.keyJSON, infos.keyPass = "", ""
+					}
+				}
+			}
+			// Clique based signers need a keyfile and unlock password, ask if unavailable
+			if infos.keyJSON == "" {
+				fmt.Println()
+				fmt.Println("Please paste the signer's key JSON:")
+				infos.keyJSON = w.readJSON()
+
+				fmt.Println()
+				fmt.Println("What's the unlock password for the account? (won't be echoed)")
+				infos.keyPass = w.readPassword()
+
+				if _, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
+					log.Error("Failed to decrypt key with given passphrase")
+					return
+				}
+			}
+		}
+	}
+	// Try to deploy the full node on the host
+	if out, err := deployNode(client, w.network, w.conf.bootFull, infos); err != nil {
+		log.Error("Failed to deploy Ethereum node container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	log.Info("Waiting for node to finish booting")
+	time.Sleep(3 * time.Second)
+
+	w.networkStats(false)
+}

cmd/rlpdump/main.go

@@ -32,6 +32,7 @@ import (
 var (
 	hexMode = flag.String("hex", "", "dump given hex data")
 	noASCII = flag.Bool("noascii", false, "don't print ASCII strings readably")
+	single  = flag.Bool("single", false, "print only the first element, discard the rest")
 )
 
 func init() {
@@ -82,6 +83,9 @@ func main() {
 			break
 		}
 		fmt.Println()
+		if *single {
+			break
+		}
 	}
 }
 

cmd/swarm/cleandb.go

@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
+// go-ethereum is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/swarm/list.go

@@ -0,0 +1,61 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/expanse-org/go-expanse/cmd/utils"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	list, err := client.List(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}

cmd/swarm/main.go

@@ -39,19 +39,16 @@ import (
 	"github.com/expanse-org/go-expanse/node"
 	"github.com/expanse-org/go-expanse/p2p"
 	"github.com/expanse-org/go-expanse/p2p/discover"
+	"github.com/expanse-org/go-expanse/params"
 	"github.com/expanse-org/go-expanse/swarm"
 	bzzapi "github.com/expanse-org/go-expanse/swarm/api"
 	"gopkg.in/urfave/cli.v1"
 )
 
-const (
-	clientIdentifier = "swarm"
-	versionString    = "0.2"
-)
+const clientIdentifier = "swarm"
 
 var (
 	gitCommit        string // Git SHA1 commit hash of the release (set via linker flags)
-	app              = utils.NewApp(gitCommit, "Expanse Swarm")
 	testbetBootNodes = []string{
 		"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
 		"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
@@ -112,19 +109,36 @@ var (
 		Name:  "defaultpath",
 		Usage: "path to file served for empty url path (none)",
 	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "force mime type",
+	}
 	CorsStringFlag = cli.StringFlag{
 		Name:  "corsdomain",
 		Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
 	}
 )
 
-func init() {
-	// Override flag defaults so bzzd can run alongside gexp.
-	utils.ListenPortFlag.Value = 30399
-	utils.IPCPathFlag.Value = utils.DirectoryString{Value: "bzzd.ipc"}
-	utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, swarmfs, web3"
+var defaultNodeConfig = node.DefaultConfig
 
-	// Set up the cli app.
+// This init function sets defaults so cmd/swarm can run alongside geth.
+func init() {
+	defaultNodeConfig.Name = clientIdentifier
+	defaultNodeConfig.Version = params.VersionWithCommit(gitCommit)
+	defaultNodeConfig.P2P.ListenAddr = ":30399"
+	defaultNodeConfig.IPCPath = "bzzd.ipc"
+	// Set flag defaults for --help display.
+	utils.ListenPortFlag.Value = 30399
+}
+
+var app = utils.NewApp(gitCommit, "Ethereum Swarm")
+
+// This init function creates the cli.App.
+func init() {
 	app.Action = bzzd
 	app.HideVersion = true // we have a command to print the version
 	app.Copyright = "Copyright 2013-2016 The go-ethereum / go-expanse Authors"
@@ -145,6 +159,15 @@ The output of this command is supposed to be machine-readable.
 			ArgsUsage: " <file>",
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
 `,
 		},
 		{
@@ -218,8 +241,8 @@ Cleans database of corrupted entries.
 		utils.MaxPeersFlag,
 		utils.NATFlag,
 		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
 		utils.IPCPathFlag,
+		utils.PasswordFileFlag,
 		// bzzd-specific flags
 		CorsStringFlag,
 		EthAPIFlag,
@@ -235,6 +258,8 @@ Cleans database of corrupted entries.
 		SwarmRecursiveUploadFlag,
 		SwarmWantManifestFlag,
 		SwarmUploadDefaultPath,
+		SwarmUpFromStdinFlag,
+		SwarmUploadMimeType,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Before = func(ctx *cli.Context) error {
@@ -256,7 +281,7 @@ func main() {
 
 func version(ctx *cli.Context) error {
 	fmt.Println(strings.Title(clientIdentifier))
-	fmt.Println("Version:", versionString)
+	fmt.Println("Version:", params.Version)
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
@@ -269,9 +294,16 @@ func version(ctx *cli.Context) error {
 }
 
 func bzzd(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	cfg := defaultNodeConfig
+	utils.SetNodeConfig(ctx, &cfg)
+	stack, err := node.New(&cfg)
+	if err != nil {
+		utils.Fatalf("can't create node: %v", err)
+	}
+
 	registerBzzService(ctx, stack)
 	utils.StartNode(stack)
+
 	go func() {
 		sigc := make(chan os.Signal, 1)
 		signal.Notify(sigc, syscall.SIGTERM)
@@ -280,6 +312,7 @@ func bzzd(ctx *cli.Context) error {
 		log.Info("Got sigterm, shutting swarm down...")
 		stack.Stop()
 	}()
+
 	networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
 	// Add bootnodes as initial peers.
 	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
@@ -296,7 +329,6 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
-
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -326,6 +358,8 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 			if err != nil {
 				utils.Fatalf("Can't connect: %v", err)
 			}
+		} else {
+			swapEnabled = false
 		}
 		return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled, cors)
 	}
@@ -349,10 +383,10 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	am := stack.AccountManager()
 	ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	return decryptStoreAccount(ks, keyid)
+	return decryptStoreAccount(ks, keyid, utils.MakePasswordList(ctx))
 }
 
-func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKey {
+func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
 	var a accounts.Account
 	var err error
 	if common.IsHexAddress(account) {
@@ -373,9 +407,9 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	if err != nil {
 		utils.Fatalf("Can't load swarm account key: %v", err)
 	}
-	for i := 1; i <= 3; i++ {
-		passphrase := promptPassphrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i))
-		key, err := keystore.DecryptKey(keyjson, passphrase)
+	for i := 0; i < 3; i++ {
+		password := getPassPhrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i+1), i, passwords)
+		key, err := keystore.DecryptKey(keyjson, password)
 		if err == nil {
 			return key.PrivateKey
 		}
@@ -384,7 +418,18 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	return nil
 }
 
-func promptPassphrase(prompt string) string {
+// getPassPhrase retrieves the password associated with bzz account, either by fetching
+// from a list of pre-loaded passwords, or by requesting it interactively from user.
+func getPassPhrase(prompt string, i int, passwords []string) string {
+	// non-interactive
+	if len(passwords) > 0 {
+		if i < len(passwords) {
+			return passwords[i]
+		}
+		return passwords[len(passwords)-1]
+	}
+
+	// fallback to interactive mode
 	if prompt != "" {
 		fmt.Println(prompt)
 	}

cmd/swarm/manifest.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -25,6 +25,8 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/swarm/api"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -41,7 +43,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 
 	if len(args) > 3 {
@@ -75,7 +77,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -105,7 +107,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -123,21 +125,17 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = &client{api: bzzapi}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		client           = swarm.NewClient(bzzapi)
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.downloadManifest(hash)
+	_, err = client.DownloadManifest(hash)
 	if err != nil {
 		utils.Fatalf("Hash to add is not present: %v", err)
 	}
@@ -162,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -172,15 +170,15 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := manifestEntry{
-			Path:        path,
+		newEntry := api.ManifestEntry{
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -191,21 +189,13 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = &client{api: bzzapi}
-		newEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		newEntry         = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -236,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -249,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := manifestEntry{
-					Path:        entry.Path,
+				myEntry := api.ManifestEntry{
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -265,7 +255,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -275,21 +265,13 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = &client{api: bzzapi}
-		entryToRemove = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		entryToRemove    = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -318,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -330,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -339,7 +321,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}

cmd/swarm/upload.go

@@ -18,8 +18,6 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -32,58 +30,83 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
-	"github.com/expanse-org/go-expanse/log"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
 func upload(ctx *cli.Context) {
+
 	args := ctx.Args()
 	var (
 		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		recursive    = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
+		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
+
 	if len(args) != 1 {
-		utils.Fatalf("Need filename as the first and only argument")
+		if fromStdin {
+			tmp, err := ioutil.TempFile("", "swarm-stdin")
+			if err != nil {
+				utils.Fatalf("error create tempfile: %s", err)
+			}
+			defer os.Remove(tmp.Name())
+			n, err := io.Copy(tmp, os.Stdin)
+			if err != nil {
+				utils.Fatalf("error copying stdin to tempfile: %s", err)
+			} else if n == 0 {
+				utils.Fatalf("error reading from stdin: zero length")
+			}
+			file = tmp.Name()
+		} else {
+			utils.Fatalf("Need filename as the first and only argument")
+		}
+	} else {
+		file = expandPath(args[0])
 	}
 
-	var (
-		file   = args[0]
-		client = &client{api: bzzapi}
-	)
-	fi, err := os.Stat(expandPath(file))
-	if err != nil {
-		utils.Fatalf("Failed to stat file: %v", err)
+	if !wantManifest {
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
+		return
 	}
-	if fi.IsDir() {
+
+	stat, err := os.Stat(file)
+	if err != nil {
+		utils.Fatalf("Error opening file: %s", err)
+	}
+	var hash string
+	if stat.IsDir() {
 		if !recursive {
 			utils.Fatalf("Argument is a directory and recursive upload is disabled")
 		}
-		if !wantManifest {
-			utils.Fatalf("Manifest is required for directory uploads")
+		hash, err = client.UploadDirectory(file, defaultPath, "")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
 		}
-		mhash, err := client.uploadDirectory(file, defaultPath)
+		f, err := swarm.Open(file)
 		if err != nil {
-			utils.Fatalf("Failed to upload directory: %v", err)
+			utils.Fatalf("Error opening file: %s", err)
 		}
-		fmt.Println(mhash)
-		return
+		defer f.Close()
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
 	}
-	entry, err := client.uploadFile(file, fi)
 	if err != nil {
-		utils.Fatalf("Upload failed: %v", err)
-	}
-	mroot := manifest{[]manifestEntry{entry}}
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
-	hash, err := client.uploadManifest(mroot)
-	if err != nil {
-		utils.Fatalf("Manifest upload failed: %v", err)
+		utils.Fatalf("Upload failed: %s", err)
 	}
 	fmt.Println(hash)
 }
@@ -112,147 +135,18 @@ func homeDir() string {
 	return ""
 }
 
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
-	api string
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
-	Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
-	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
-	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
-	mroot := manifest{}
-	req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
-	if err != nil {
-		return mroot, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return mroot, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-
-	err = json.Unmarshal(content, &mroot)
-	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
-	}
-	return mroot, err
+func detectMimeType(file string) string {
+	if ext := filepath.Ext(file); ext != "" {
+		return mime.TypeByExtension(ext)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	buf := make([]byte, 512)
+	if n, _ := f.Read(buf); n > 0 {
+		return http.DetectContentType(buf)
+	}
+	return ""
 }

cmd/utils/customflags.go

@@ -17,6 +17,7 @@
 package utils
 
 import (
+	"encoding"
 	"errors"
 	"flag"
 	"fmt"
@@ -78,6 +79,58 @@ func (self DirectoryFlag) Apply(set *flag.FlagSet) {
 	})
 }
 
+type TextMarshaler interface {
+	encoding.TextMarshaler
+	encoding.TextUnmarshaler
+}
+
+// textMarshalerVal turns a TextMarshaler into a flag.Value
+type textMarshalerVal struct {
+	v TextMarshaler
+}
+
+func (v textMarshalerVal) String() string {
+	if v.v == nil {
+		return ""
+	}
+	text, _ := v.v.MarshalText()
+	return string(text)
+}
+
+func (v textMarshalerVal) Set(s string) error {
+	return v.v.UnmarshalText([]byte(s))
+}
+
+// TextMarshalerFlag wraps a TextMarshaler value.
+type TextMarshalerFlag struct {
+	Name  string
+	Value TextMarshaler
+	Usage string
+}
+
+func (f TextMarshalerFlag) GetName() string {
+	return f.Name
+}
+
+func (f TextMarshalerFlag) String() string {
+	return fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+}
+
+func (f TextMarshalerFlag) Apply(set *flag.FlagSet) {
+	eachName(f.Name, func(name string) {
+		set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage)
+	})
+}
+
+// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set.
+func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler {
+	val := ctx.GlobalGeneric(name)
+	if val == nil {
+		return nil
+	}
+	return val.(textMarshalerVal).v
+}
+
 // BigFlag is a command line flag that accepts 256 bit big integers in decimal or
 // hexadecimal syntax.
 type BigFlag struct {

cmd/utils/flags.go

@@ -23,7 +23,6 @@ import (
 	"io/ioutil"
 	"math/big"
 	"os"
-	"os/user"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -32,11 +31,14 @@ import (
 	"github.com/expanse-org/go-expanse/accounts"
 	"github.com/expanse-org/go-expanse/accounts/keystore"
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/crypto"
 	"github.com/expanse-org/go-expanse/eth"
+	"github.com/expanse-org/go-expanse/eth/downloader"
+	"github.com/expanse-org/go-expanse/eth/gasprice"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/ethstats"
 	"github.com/expanse-org/go-expanse/event"
@@ -44,13 +46,12 @@ import (
 	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/metrics"
 	"github.com/expanse-org/go-expanse/node"
+	"github.com/expanse-org/go-expanse/p2p"
 	"github.com/expanse-org/go-expanse/p2p/discover"
 	"github.com/expanse-org/go-expanse/p2p/discv5"
 	"github.com/expanse-org/go-expanse/p2p/nat"
 	"github.com/expanse-org/go-expanse/p2p/netutil"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
-	"github.com/expanse-org/go-expanse/rpc"
 	whisper "github.com/expanse-org/go-expanse/whisper/whisperv2"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -121,35 +122,36 @@ var (
 	EthashCachesInMemoryFlag = cli.IntFlag{
 		Name:  "ethash.cachesinmem",
 		Usage: "Number of recent ethash caches to keep in memory (16MB each)",
-		Value: 2,
+		Value: eth.DefaultConfig.EthashCachesInMem,
 	}
 	EthashCachesOnDiskFlag = cli.IntFlag{
 		Name:  "ethash.cachesondisk",
 		Usage: "Number of recent ethash caches to keep on disk (16MB each)",
-		Value: 3,
+		Value: eth.DefaultConfig.EthashCachesOnDisk,
 	}
 	EthashDatasetDirFlag = DirectoryFlag{
 		Name:  "ethash.dagdir",
 		Usage: "Directory to store the ethash mining DAGs (default = inside home folder)",
+		Value: DirectoryString{eth.DefaultConfig.EthashDatasetDir},
 	}
 	EthashDatasetsInMemoryFlag = cli.IntFlag{
 		Name:  "ethash.dagsinmem",
 		Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)",
-		Value: 1,
+		Value: eth.DefaultConfig.EthashDatasetsInMem,
 	}
 	EthashDatasetsOnDiskFlag = cli.IntFlag{
 		Name:  "ethash.dagsondisk",
 		Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)",
-		Value: 2,
+		Value: eth.DefaultConfig.EthashDatasetsOnDisk,
 	}
 	NetworkIdFlag = cli.IntFlag{
 		Name:  "networkid",
 		Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten)",
-		Value: eth.NetworkId,
+		Value: eth.DefaultConfig.NetworkId,
 	}
 	TestNetFlag = cli.BoolFlag{
 		Name:  "testnet",
-		Usage: "Ropsten network: pre-configured test network",
+		Usage: "Ropsten network: pre-configured proof-of-work test network",
 	}
 	DevModeFlag = cli.BoolFlag{
 		Name:  "dev",
@@ -172,6 +174,13 @@ var (
 		Name:  "light",
 		Usage: "Enable light client mode",
 	}
+	defaultSyncMode = eth.DefaultConfig.SyncMode
+	SyncModeFlag    = TextMarshalerFlag{
+		Name:  "syncmode",
+		Usage: `Blockchain sync mode ("fast", "full", or "light")`,
+		Value: &defaultSyncMode,
+	}
+
 	LightServFlag = cli.IntFlag{
 		Name:  "lightserv",
 		Usage: "Maximum percentage of time allowed for serving LES requests (0-90)",
@@ -238,19 +247,6 @@ var (
 		Value: "",
 	}
 
-	VMForceJitFlag = cli.BoolFlag{
-		Name:  "forcejit",
-		Usage: "Force the JIT VM to take precedence",
-	}
-	VMJitCacheFlag = cli.IntFlag{
-		Name:  "jitcache",
-		Usage: "Amount of cached JIT VM programs",
-		Value: 64,
-	}
-	VMEnableJitFlag = cli.BoolFlag{
-		Name:  "jitvm",
-		Usage: "Enable the JIT VM",
-	}
 	VMEnableDebugFlag = cli.BoolFlag{
 		Name:  "vmdebug",
 		Usage: "Record information useful for VM and contract debugging",
@@ -295,21 +291,15 @@ var (
 	RPCApiFlag = cli.StringFlag{
 		Name:  "rpcapi",
 		Usage: "API's offered over the HTTP-RPC interface",
-		Value: rpc.DefaultHTTPApis,
+		Value: "",
 	}
 	IPCDisabledFlag = cli.BoolFlag{
 		Name:  "ipcdisable",
 		Usage: "Disable the IPC-RPC server",
 	}
-	IPCApiFlag = cli.StringFlag{
-		Name:  "ipcapi",
-		Usage: "APIs offered over the IPC-RPC interface",
-		Value: rpc.DefaultIPCApis,
-	}
 	IPCPathFlag = DirectoryFlag{
 		Name:  "ipcpath",
 		Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)",
-		Value: DirectoryString{"gexp.ipc"},
 	}
 	WSEnabledFlag = cli.BoolFlag{
 		Name:  "ws",
@@ -328,7 +318,7 @@ var (
 	WSApiFlag = cli.StringFlag{
 		Name:  "wsapi",
 		Usage: "API's offered over the WS-RPC interface",
-		Value: rpc.DefaultHTTPApis,
+		Value: "",
 	}
 	WSAllowedOriginsFlag = cli.StringFlag{
 		Name:  "wsorigins",
@@ -402,42 +392,17 @@ var (
 		Usage: "JavaScript root path for `loadScript`",
 		Value: ".",
 	}
-	SolcPathFlag = cli.StringFlag{
-		Name:  "solc",
-		Usage: "Solidity compiler command to be used",
-		Value: "solc",
-	}
 
 	// Gas price oracle settings
-	GpoMinGasPriceFlag = BigFlag{
-		Name:  "gpomin",
-		Usage: "Minimum suggested gas price",
-		Value: big.NewInt(20 * params.Shannon),
+	GpoBlocksFlag = cli.IntFlag{
+		Name:  "gpoblocks",
+		Usage: "Number of recent blocks to check for gas prices",
+		Value: eth.DefaultConfig.GPO.Blocks,
 	}
-	GpoMaxGasPriceFlag = BigFlag{
-		Name:  "gpomax",
-		Usage: "Maximum suggested gas price",
-		Value: big.NewInt(500 * params.Shannon),
-	}
-	GpoFullBlockRatioFlag = cli.IntFlag{
-		Name:  "gpofull",
-		Usage: "Full block threshold for gas price calculation (%)",
-		Value: 80,
-	}
-	GpobaseStepDownFlag = cli.IntFlag{
-		Name:  "gpobasedown",
-		Usage: "Suggested gas price base step down ratio (1/1000)",
-		Value: 10,
-	}
-	GpobaseStepUpFlag = cli.IntFlag{
-		Name:  "gpobaseup",
-		Usage: "Suggested gas price base step up ratio (1/1000)",
-		Value: 100,
-	}
-	GpobaseCorrectionFactorFlag = cli.IntFlag{
-		Name:  "gpobasecf",
-		Usage: "Suggested gas price base correction factor (%)",
-		Value: 110,
+	GpoPercentileFlag = cli.IntFlag{
+		Name:  "gpopercentile",
+		Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices",
+		Value: eth.DefaultConfig.GPO.Percentile,
 	}
 )
 
@@ -456,88 +421,42 @@ func MakeDataDir(ctx *cli.Context) string {
 	return ""
 }
 
-// MakeEthashCacheDir returns the directory to use for storing the ethash cache
-// dumps.
-func MakeEthashCacheDir(ctx *cli.Context) string {
-	if ctx.GlobalIsSet(EthashCacheDirFlag.Name) && ctx.GlobalString(EthashCacheDirFlag.Name) == "" {
-		return ""
-	}
-	if !ctx.GlobalIsSet(EthashCacheDirFlag.Name) {
-		return "ethash"
-	}
-	return ctx.GlobalString(EthashCacheDirFlag.Name)
-}
-
-// MakeEthashDatasetDir returns the directory to use for storing the full ethash
-// dataset dumps.
-func MakeEthashDatasetDir(ctx *cli.Context) string {
-	if !ctx.GlobalIsSet(EthashDatasetDirFlag.Name) {
-		home := os.Getenv("HOME")
-		if home == "" {
-			if user, err := user.Current(); err == nil {
-				home = user.HomeDir
-			}
-		}
-		if runtime.GOOS == "windows" {
-			return filepath.Join(home, "AppData", "Ethash")
-		}
-		return filepath.Join(home, ".ethash")
-	}
-	return ctx.GlobalString(EthashDatasetDirFlag.Name)
-}
-
-// MakeIPCPath creates an IPC path configuration from the set command line flags,
-// returning an empty string if IPC was explicitly disabled, or the set path.
-func MakeIPCPath(ctx *cli.Context) string {
-	if ctx.GlobalBool(IPCDisabledFlag.Name) {
-		return ""
-	}
-	return ctx.GlobalString(IPCPathFlag.Name)
-}
-
-// MakeNodeKey creates a node key from set command line flags, either loading it
+// setNodeKey creates a node key from set command line flags, either loading it
 // from a file or as a specified hex value. If neither flags were provided, this
 // method returns nil and an emphemeral key is to be generated.
-func MakeNodeKey(ctx *cli.Context) *ecdsa.PrivateKey {
+func setNodeKey(ctx *cli.Context, cfg *p2p.Config) {
 	var (
 		hex  = ctx.GlobalString(NodeKeyHexFlag.Name)
 		file = ctx.GlobalString(NodeKeyFileFlag.Name)
-
-		key *ecdsa.PrivateKey
-		err error
+		key  *ecdsa.PrivateKey
+		err  error
 	)
 	switch {
 	case file != "" && hex != "":
 		Fatalf("Options %q and %q are mutually exclusive", NodeKeyFileFlag.Name, NodeKeyHexFlag.Name)
-
 	case file != "":
 		if key, err = crypto.LoadECDSA(file); err != nil {
 			Fatalf("Option %q: %v", NodeKeyFileFlag.Name, err)
 		}
-
+		cfg.PrivateKey = key
 	case hex != "":
 		if key, err = crypto.HexToECDSA(hex); err != nil {
 			Fatalf("Option %q: %v", NodeKeyHexFlag.Name, err)
 		}
+		cfg.PrivateKey = key
 	}
-	return key
 }
 
-// makeNodeUserIdent creates the user identifier from CLI flags.
-func makeNodeUserIdent(ctx *cli.Context) string {
-	var comps []string
+// setNodeUserIdent creates the user identifier from CLI flags.
+func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) {
 	if identity := ctx.GlobalString(IdentityFlag.Name); len(identity) > 0 {
-		comps = append(comps, identity)
+		cfg.UserIdent = identity
 	}
-	if ctx.GlobalBool(VMEnableJitFlag.Name) {
-		comps = append(comps, "JIT")
-	}
-	return strings.Join(comps, "/")
 }
 
-// MakeBootstrapNodes creates a list of bootstrap nodes from the command line
+// setBootstrapNodes creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
-func MakeBootstrapNodes(ctx *cli.Context) []*discover.Node {
+func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 	urls := params.MainnetBootnodes
 	if ctx.GlobalIsSet(BootnodesFlag.Name) {
 		urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
@@ -545,62 +464,68 @@ func MakeBootstrapNodes(ctx *cli.Context) []*discover.Node {
 		urls = params.TestnetBootnodes
 	}
 
-	bootnodes := make([]*discover.Node, 0, len(urls))
+	cfg.BootstrapNodes = make([]*discover.Node, 0, len(urls))
 	for _, url := range urls {
 		node, err := discover.ParseNode(url)
 		if err != nil {
 			log.Error("Bootstrap URL invalid", "enode", url, "err", err)
 			continue
 		}
-		bootnodes = append(bootnodes, node)
+		cfg.BootstrapNodes = append(cfg.BootstrapNodes, node)
 	}
-	return bootnodes
 }
 
-// MakeBootstrapNodesV5 creates a list of bootstrap nodes from the command line
+// setBootstrapNodesV5 creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
-func MakeBootstrapNodesV5(ctx *cli.Context) []*discv5.Node {
+func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
 	urls := params.DiscoveryV5Bootnodes
 	if ctx.GlobalIsSet(BootnodesFlag.Name) {
 		urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
+	} else if cfg.BootstrapNodesV5 == nil {
+		return // already set, don't apply defaults.
 	}
 
-	bootnodes := make([]*discv5.Node, 0, len(urls))
+	cfg.BootstrapNodesV5 = make([]*discv5.Node, 0, len(urls))
 	for _, url := range urls {
 		node, err := discv5.ParseNode(url)
 		if err != nil {
 			log.Error("Bootstrap URL invalid", "enode", url, "err", err)
 			continue
 		}
-		bootnodes = append(bootnodes, node)
+		cfg.BootstrapNodesV5 = append(cfg.BootstrapNodesV5, node)
 	}
-	return bootnodes
 }
 
-// MakeListenAddress creates a TCP listening address string from set command
+// setListenAddress creates a TCP listening address string from set command
 // line flags.
-func MakeListenAddress(ctx *cli.Context) string {
-	return fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name))
-}
-
-// MakeDiscoveryV5Address creates a UDP listening address string from set command
-// line flags for the V5 discovery protocol.
-func MakeDiscoveryV5Address(ctx *cli.Context) string {
-	return fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)+1)
-}
-
-// MakeNAT creates a port mapper from set command line flags.
-func MakeNAT(ctx *cli.Context) nat.Interface {
-	natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name))
-	if err != nil {
-		Fatalf("Option %s: %v", NATFlag.Name, err)
+func setListenAddress(ctx *cli.Context, cfg *p2p.Config) {
+	if ctx.GlobalIsSet(ListenPortFlag.Name) {
+		cfg.ListenAddr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name))
 	}
-	return natif
 }
 
-// MakeRPCModules splits input separated by a comma and trims excessive white
-// space from the substrings.
-func MakeRPCModules(input string) []string {
+// setDiscoveryV5Address creates a UDP listening address string from set command
+// line flags for the V5 discovery protocol.
+func setDiscoveryV5Address(ctx *cli.Context, cfg *p2p.Config) {
+	if ctx.GlobalIsSet(ListenPortFlag.Name) {
+		cfg.DiscoveryV5Addr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)+1)
+	}
+}
+
+// setNAT creates a port mapper from command line flags.
+func setNAT(ctx *cli.Context, cfg *p2p.Config) {
+	if ctx.GlobalIsSet(NATFlag.Name) {
+		natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name))
+		if err != nil {
+			Fatalf("Option %s: %v", NATFlag.Name, err)
+		}
+		cfg.NAT = natif
+	}
+}
+
+// splitAndTrim splits input separated by a comma
+// and trims excessive white space from the substrings.
+func splitAndTrim(input string) []string {
 	result := strings.Split(input, ",")
 	for i, r := range result {
 		result[i] = strings.TrimSpace(r)
@@ -608,27 +533,63 @@ func MakeRPCModules(input string) []string {
 	return result
 }
 
-// MakeHTTPRpcHost creates the HTTP RPC listener interface string from the set
+// setHTTP creates the HTTP RPC listener interface string from the set
 // command line flags, returning empty if the HTTP endpoint is disabled.
-func MakeHTTPRpcHost(ctx *cli.Context) string {
-	if !ctx.GlobalBool(RPCEnabledFlag.Name) {
-		return ""
+func setHTTP(ctx *cli.Context, cfg *node.Config) {
+	if ctx.GlobalBool(RPCEnabledFlag.Name) && cfg.HTTPHost == "" {
+		cfg.HTTPHost = "127.0.0.1"
+		if ctx.GlobalIsSet(RPCListenAddrFlag.Name) {
+			cfg.HTTPHost = ctx.GlobalString(RPCListenAddrFlag.Name)
+		}
+	}
+
+	if ctx.GlobalIsSet(RPCPortFlag.Name) {
+		cfg.HTTPPort = ctx.GlobalInt(RPCPortFlag.Name)
+	}
+	if ctx.GlobalIsSet(RPCCORSDomainFlag.Name) {
+		cfg.HTTPCors = splitAndTrim(ctx.GlobalString(RPCCORSDomainFlag.Name))
+	}
+	if ctx.GlobalIsSet(RPCApiFlag.Name) {
+		cfg.HTTPModules = splitAndTrim(ctx.GlobalString(RPCApiFlag.Name))
 	}
-	return ctx.GlobalString(RPCListenAddrFlag.Name)
 }
 
-// MakeWSRpcHost creates the WebSocket RPC listener interface string from the set
+// setWS creates the WebSocket RPC listener interface string from the set
 // command line flags, returning empty if the HTTP endpoint is disabled.
-func MakeWSRpcHost(ctx *cli.Context) string {
-	if !ctx.GlobalBool(WSEnabledFlag.Name) {
-		return ""
+func setWS(ctx *cli.Context, cfg *node.Config) {
+	if ctx.GlobalBool(WSEnabledFlag.Name) && cfg.WSHost == "" {
+		cfg.WSHost = "127.0.0.1"
+		if ctx.GlobalIsSet(WSListenAddrFlag.Name) {
+			cfg.WSHost = ctx.GlobalString(WSListenAddrFlag.Name)
+		}
+	}
+
+	if ctx.GlobalIsSet(WSPortFlag.Name) {
+		cfg.WSPort = ctx.GlobalInt(WSPortFlag.Name)
+	}
+	if ctx.GlobalIsSet(WSAllowedOriginsFlag.Name) {
+		cfg.WSOrigins = splitAndTrim(ctx.GlobalString(WSAllowedOriginsFlag.Name))
+	}
+	if ctx.GlobalIsSet(WSApiFlag.Name) {
+		cfg.WSModules = splitAndTrim(ctx.GlobalString(WSApiFlag.Name))
 	}
-	return ctx.GlobalString(WSListenAddrFlag.Name)
 }
 
-// MakeDatabaseHandles raises out the number of allowed file handles per process
-// for Gexp and returns half of the allowance to assign to the database.
-func MakeDatabaseHandles() int {
+// setIPC creates an IPC path configuration from the set command line flags,
+// returning an empty string if IPC was explicitly disabled, or the set path.
+func setIPC(ctx *cli.Context, cfg *node.Config) {
+	checkExclusive(ctx, IPCDisabledFlag, IPCPathFlag)
+	switch {
+	case ctx.GlobalBool(IPCDisabledFlag.Name):
+		cfg.IPCPath = ""
+	case ctx.GlobalIsSet(IPCPathFlag.Name):
+		cfg.IPCPath = ctx.GlobalString(IPCPathFlag.Name)
+	}
+}
+
+// makeDatabaseHandles raises out the number of allowed file handles per process
+// for Geth and returns half of the allowance to assign to the database.
+func makeDatabaseHandles() int {
 	if err := raiseFdLimit(2048); err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
@@ -661,33 +622,25 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error
 	return accs[index], nil
 }
 
-// MakeEtherbase retrieves the etherbase either from the directly specified
+// setEtherbase retrieves the etherbase either from the directly specified
 // command line flags or from the keystore if CLI indexed.
-func MakeEtherbase(ks *keystore.KeyStore, ctx *cli.Context) common.Address {
+func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) {
+	if ctx.GlobalIsSet(EtherbaseFlag.Name) {
+		account, err := MakeAddress(ks, ctx.GlobalString(EtherbaseFlag.Name))
+		if err != nil {
+			Fatalf("Option %q: %v", EtherbaseFlag.Name, err)
+		}
+		cfg.Etherbase = account.Address
+		return
+	}
 	accounts := ks.Accounts()
-	if !ctx.GlobalIsSet(EtherbaseFlag.Name) && len(accounts) == 0 {
-		log.Warn("No etherbase set and no accounts found as default")
-		return common.Address{}
+	if (cfg.Etherbase == common.Address{}) {
+		if len(accounts) > 0 {
+			cfg.Etherbase = accounts[0].Address
+		} else {
+			log.Warn("No etherbase set and no accounts found as default")
+		}
 	}
-	etherbase := ctx.GlobalString(EtherbaseFlag.Name)
-	if etherbase == "" {
-		return common.Address{}
-	}
-	// If the specified etherbase is a valid address, return it
-	account, err := MakeAddress(ks, etherbase)
-	if err != nil {
-		Fatalf("Option %q: %v", EtherbaseFlag.Name, err)
-	}
-	return account.Address
-}
-
-// MakeMinerExtra resolves extradata for the miner from the set command line flags
-// or returns a default one composed on the client, runtime and OS metadata.
-func MakeMinerExtra(extra []byte, ctx *cli.Context) []byte {
-	if ctx.GlobalIsSet(ExtraDataFlag.Name) {
-		return []byte(ctx.GlobalString(ExtraDataFlag.Name))
-	}
-	return extra
 }
 
 // MakePasswordList reads password lines from the file specified by --password.
@@ -708,148 +661,213 @@ func MakePasswordList(ctx *cli.Context) []string {
 	return lines
 }
 
-// MakeNode configures a node with no services from command line flags.
-func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
-	vsn := params.Version
-	if gitCommit != "" {
-		vsn += "-" + gitCommit[:8]
+func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
+	setNodeKey(ctx, cfg)
+	setNAT(ctx, cfg)
+	setListenAddress(ctx, cfg)
+	setDiscoveryV5Address(ctx, cfg)
+	setBootstrapNodes(ctx, cfg)
+	setBootstrapNodesV5(ctx, cfg)
+
+	if ctx.GlobalIsSet(MaxPeersFlag.Name) {
+		cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
+	}
+	if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) {
+		cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name)
+	}
+	if ctx.GlobalIsSet(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name) {
+		cfg.NoDiscovery = true
 	}
 
-	// if we're running a light client or server, force enable the v5 peer discovery unless it is explicitly disabled with --nodiscover
-	// note that explicitly specifying --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
+	// if we're running a light client or server, force enable the v5 peer discovery
+	// unless it is explicitly disabled with --nodiscover note that explicitly specifying
+	// --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
 	forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name)
+	if ctx.GlobalIsSet(DiscoveryV5Flag.Name) {
+		cfg.DiscoveryV5 = ctx.GlobalBool(DiscoveryV5Flag.Name)
+	} else if forceV5Discovery {
+		cfg.DiscoveryV5 = true
+	}
 
-	config := &node.Config{
-		DataDir:           MakeDataDir(ctx),
-		KeyStoreDir:       ctx.GlobalString(KeyStoreDirFlag.Name),
-		UseLightweightKDF: ctx.GlobalBool(LightKDFFlag.Name),
-		PrivateKey:        MakeNodeKey(ctx),
-		Name:              name,
-		Version:           vsn,
-		UserIdent:         makeNodeUserIdent(ctx),
-		NoDiscovery:       ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name), // always disable v4 discovery in light client mode
-		DiscoveryV5:       ctx.GlobalBool(DiscoveryV5Flag.Name) || forceV5Discovery,
-		DiscoveryV5Addr:   MakeDiscoveryV5Address(ctx),
-		BootstrapNodes:    MakeBootstrapNodes(ctx),
-		BootstrapNodesV5:  MakeBootstrapNodesV5(ctx),
-		ListenAddr:        MakeListenAddress(ctx),
-		NAT:               MakeNAT(ctx),
-		MaxPeers:          ctx.GlobalInt(MaxPeersFlag.Name),
-		MaxPendingPeers:   ctx.GlobalInt(MaxPendingPeersFlag.Name),
-		IPCPath:           MakeIPCPath(ctx),
-		HTTPHost:          MakeHTTPRpcHost(ctx),
-		HTTPPort:          ctx.GlobalInt(RPCPortFlag.Name),
-		HTTPCors:          ctx.GlobalString(RPCCORSDomainFlag.Name),
-		HTTPModules:       MakeRPCModules(ctx.GlobalString(RPCApiFlag.Name)),
-		WSHost:            MakeWSRpcHost(ctx),
-		WSPort:            ctx.GlobalInt(WSPortFlag.Name),
-		WSOrigins:         ctx.GlobalString(WSAllowedOriginsFlag.Name),
-		WSModules:         MakeRPCModules(ctx.GlobalString(WSApiFlag.Name)),
-	}
-	if ctx.GlobalBool(DevModeFlag.Name) {
-		if !ctx.GlobalIsSet(DataDirFlag.Name) {
-			config.DataDir = filepath.Join(os.TempDir(), "/ethereum_dev_mode")
-		}
-		// --dev mode does not need p2p networking.
-		config.MaxPeers = 0
-		config.ListenAddr = ":0"
-	}
 	if netrestrict := ctx.GlobalString(NetrestrictFlag.Name); netrestrict != "" {
 		list, err := netutil.ParseNetlist(netrestrict)
 		if err != nil {
 			Fatalf("Option %q: %v", NetrestrictFlag.Name, err)
 		}
-		config.NetRestrict = list
+		cfg.NetRestrict = list
 	}
 
-	stack, err := node.New(config)
-	if err != nil {
-		Fatalf("Failed to create the protocol stack: %v", err)
+	if ctx.GlobalBool(DevModeFlag.Name) {
+		// --dev mode can't use p2p networking.
+		cfg.MaxPeers = 0
+		cfg.ListenAddr = ":0"
+		cfg.NoDiscovery = true
+		cfg.DiscoveryV5 = false
 	}
-	return stack
 }
 
-// RegisterEthService configures eth.Ethereum from command line flags and adds it to the
-// given node.
-func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
-	// Avoid conflicting network flags
-	networks, netFlags := 0, []cli.BoolFlag{DevModeFlag, TestNetFlag}
-	for _, flag := range netFlags {
-		if ctx.GlobalBool(flag.Name) {
-			networks++
+// SetNodeConfig applies node-related command line flags to the config.
+func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
+	SetP2PConfig(ctx, &cfg.P2P)
+	setIPC(ctx, cfg)
+	setHTTP(ctx, cfg)
+	setWS(ctx, cfg)
+	setNodeUserIdent(ctx, cfg)
+
+	switch {
+	case ctx.GlobalIsSet(DataDirFlag.Name):
+		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
+	case ctx.GlobalBool(DevModeFlag.Name):
+		cfg.DataDir = filepath.Join(os.TempDir(), "ethereum_dev_mode")
+	case ctx.GlobalBool(TestNetFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
+	}
+
+	if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
+		cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
+	}
+	if ctx.GlobalIsSet(LightKDFFlag.Name) {
+		cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name)
+	}
+}
+
+func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
+	if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
+		cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
+	}
+	if ctx.GlobalIsSet(GpoPercentileFlag.Name) {
+		cfg.Percentile = ctx.GlobalInt(GpoPercentileFlag.Name)
+	}
+}
+
+func setEthash(ctx *cli.Context, cfg *eth.Config) {
+	if ctx.GlobalIsSet(EthashCacheDirFlag.Name) {
+		cfg.EthashCacheDir = ctx.GlobalString(EthashCacheDirFlag.Name)
+	}
+	if ctx.GlobalIsSet(EthashDatasetDirFlag.Name) {
+		cfg.EthashDatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name)
+	}
+	if ctx.GlobalIsSet(EthashCachesInMemoryFlag.Name) {
+		cfg.EthashCachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name)
+	}
+	if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) {
+		cfg.EthashCachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
+	}
+	if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) {
+		cfg.EthashDatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name)
+	}
+	if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) {
+		cfg.EthashDatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name)
+	}
+}
+
+func checkExclusive(ctx *cli.Context, flags ...cli.Flag) {
+	set := make([]string, 0, 1)
+	for _, flag := range flags {
+		if ctx.GlobalIsSet(flag.GetName()) {
+			set = append(set, "--"+flag.GetName())
 		}
 	}
-	if networks > 1 {
-		Fatalf("The %v flags are mutually exclusive", netFlags)
+	if len(set) > 1 {
+		Fatalf("flags %v can't be used at the same time", strings.Join(set, ", "))
 	}
+}
+
+// SetEthConfig applies eth-related command line flags to the config.
+func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
+	// Avoid conflicting network flags
+	checkExclusive(ctx, DevModeFlag, TestNetFlag)
+	checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag)
+
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	setEtherbase(ctx, ks, cfg)
+	setGPO(ctx, &cfg.GPO)
+	setEthash(ctx, cfg)
 
-	ethConf := &eth.Config{
-		Etherbase:               MakeEtherbase(ks, ctx),
-		FastSync:                ctx.GlobalBool(FastSyncFlag.Name),
-		LightMode:               ctx.GlobalBool(LightModeFlag.Name),
-		LightServ:               ctx.GlobalInt(LightServFlag.Name),
-		LightPeers:              ctx.GlobalInt(LightPeersFlag.Name),
-		MaxPeers:                ctx.GlobalInt(MaxPeersFlag.Name),
-		DatabaseCache:           ctx.GlobalInt(CacheFlag.Name),
-		DatabaseHandles:         MakeDatabaseHandles(),
-		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),
-		MinerThreads:            ctx.GlobalInt(MinerThreadsFlag.Name),
-		ExtraData:               MakeMinerExtra(extra, ctx),
-		DocRoot:                 ctx.GlobalString(DocRootFlag.Name),
-		GasPrice:                GlobalBig(ctx, GasPriceFlag.Name),
-		GpoMinGasPrice:          GlobalBig(ctx, GpoMinGasPriceFlag.Name),
-		GpoMaxGasPrice:          GlobalBig(ctx, GpoMaxGasPriceFlag.Name),
-		GpoFullBlockRatio:       ctx.GlobalInt(GpoFullBlockRatioFlag.Name),
-		GpobaseStepDown:         ctx.GlobalInt(GpobaseStepDownFlag.Name),
-		GpobaseStepUp:           ctx.GlobalInt(GpobaseStepUpFlag.Name),
-		GpobaseCorrectionFactor: ctx.GlobalInt(GpobaseCorrectionFactorFlag.Name),
-		SolcPath:                ctx.GlobalString(SolcPathFlag.Name),
-		EthashCacheDir:          MakeEthashCacheDir(ctx),
-		EthashCachesInMem:       ctx.GlobalInt(EthashCachesInMemoryFlag.Name),
-		EthashCachesOnDisk:      ctx.GlobalInt(EthashCachesOnDiskFlag.Name),
-		EthashDatasetDir:        MakeEthashDatasetDir(ctx),
-		EthashDatasetsInMem:     ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name),
-		EthashDatasetsOnDisk:    ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name),
-		EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name),
+	switch {
+	case ctx.GlobalIsSet(SyncModeFlag.Name):
+		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
+	case ctx.GlobalBool(FastSyncFlag.Name):
+		cfg.SyncMode = downloader.FastSync
+	case ctx.GlobalBool(LightModeFlag.Name):
+		cfg.SyncMode = downloader.LightSync
+	}
+	if ctx.GlobalIsSet(LightServFlag.Name) {
+		cfg.LightServ = ctx.GlobalInt(LightServFlag.Name)
+	}
+	if ctx.GlobalIsSet(LightPeersFlag.Name) {
+		cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name)
+	}
+	if ctx.GlobalIsSet(NetworkIdFlag.Name) {
+		cfg.NetworkId = ctx.GlobalInt(NetworkIdFlag.Name)
 	}
 
-	// Override any default configs in dev mode or the test net
+	// Ethereum needs to know maxPeers to calculate the light server peer ratio.
+	// TODO(fjl): ensure Ethereum can get MaxPeers from node.
+	cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
+
+	if ctx.GlobalIsSet(CacheFlag.Name) {
+		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name)
+	}
+	cfg.DatabaseHandles = makeDatabaseHandles()
+
+	if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
+		cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
+	}
+	if ctx.GlobalIsSet(DocRootFlag.Name) {
+		cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)
+	}
+	if ctx.GlobalIsSet(ExtraDataFlag.Name) {
+		cfg.ExtraData = []byte(ctx.GlobalString(ExtraDataFlag.Name))
+	}
+	if ctx.GlobalIsSet(GasPriceFlag.Name) {
+		cfg.GasPrice = GlobalBig(ctx, GasPriceFlag.Name)
+	}
+	if ctx.GlobalIsSet(VMEnableDebugFlag.Name) {
+		// TODO(fjl): force-enable this in --dev mode
+		cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name)
+	}
+
+	// Override any default configs for --dev and --testnet.
 	switch {
 	case ctx.GlobalBool(TestNetFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
-			ethConf.NetworkId = 3
+			cfg.NetworkId = 3
 		}
-		ethConf.Genesis = core.DefaultTestnetGenesisBlock()
+		cfg.Genesis = core.DefaultTestnetGenesisBlock()
 	case ctx.GlobalBool(DevModeFlag.Name):
-		ethConf.Genesis = core.DevGenesisBlock()
+		cfg.Genesis = core.DevGenesisBlock()
 		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
-			ethConf.GasPrice = new(big.Int)
+			cfg.GasPrice = new(big.Int)
 		}
-		ethConf.PowTest = true
+		cfg.PowTest = true
 	}
-	// Override any global options pertaining to the Ethereum protocol
+
+	// TODO(fjl): move trie cache generations into config
 	if gen := ctx.GlobalInt(TrieCacheGenFlag.Name); gen > 0 {
 		state.MaxTrieCacheGen = uint16(gen)
 	}
+}
 
-	if ethConf.LightMode {
-		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-			return les.New(ctx, ethConf)
-		}); err != nil {
-			Fatalf("Failed to register the Expanse light node service: %v", err)
-		}
+// RegisterEthService adds an Ethereum client to the stack.
+func RegisterEthService(stack *node.Node, cfg *eth.Config) {
+	var err error
+	if cfg.SyncMode == downloader.LightSync {
+		err = stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			return les.New(ctx, cfg)
+		})
 	} else {
-		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-			fullNode, err := eth.New(ctx, ethConf)
-			if fullNode != nil && ethConf.LightServ > 0 {
-				ls, _ := les.NewLesServer(fullNode, ethConf)
+		err = stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			fullNode, err := eth.New(ctx, cfg)
+			if fullNode != nil && cfg.LightServ > 0 {
+				ls, _ := les.NewLesServer(fullNode, cfg)
 				fullNode.AddLesServer(ls)
 			}
 			return fullNode, err
-		}); err != nil {
-			Fatalf("Failed to register the Expanse full node service: %v", err)
-		}
+		})
+	}
+	if err != nil {
+		Fatalf("Failed to register the Expanse service: %v", err)
 	}
 }
 
@@ -879,6 +897,7 @@ func RegisterEthStatsService(stack *node.Node, url string) {
 
 // SetupNetwork configures the system for either the main net or some test network.
 func SetupNetwork(ctx *cli.Context) {
+	// TODO(fjl): move target gas limit into config
 	params.TargetGasLimit = new(big.Int).SetUint64(ctx.GlobalUint64(TargetGasLimitFlag.Name))
 }
 
@@ -894,7 +913,7 @@ func ChainDbName(ctx *cli.Context) string {
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	var (
 		cache   = ctx.GlobalInt(CacheFlag.Name)
-		handles = MakeDatabaseHandles()
+		handles = makeDatabaseHandles()
 		name    = ChainDbName(ctx)
 	)
 
@@ -921,16 +940,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	var err error
 	chainDb = MakeChainDatabase(ctx, stack)
 
-	seal := pow.PoW(pow.FakePow{})
+	engine := ethash.NewFaker()
 	if !ctx.GlobalBool(FakePoWFlag.Name) {
-		seal = pow.NewFullEthash("", 1, 0, "", 1, 0)
+		engine = ethash.New("", 1, 0, "", 1, 0)
 	}
 	config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
 	if err != nil {
 		Fatalf("%v", err)
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
-	chain, err = core.NewBlockChain(chainDb, config, seal, new(event.TypeMux), vmcfg)
+	chain, err = core.NewBlockChain(chainDb, config, engine, new(event.TypeMux), vmcfg)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}

cmd/wnode/main.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -27,7 +27,9 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -46,7 +48,6 @@ import (
 )
 
 const quitCommand = "~Q"
-const symKeyName = "da919ea33001b04dfc630522e33078ec0df11"
 
 // singletons
 var (
@@ -64,7 +65,8 @@ var (
 	pub        *ecdsa.PublicKey
 	asymKey    *ecdsa.PrivateKey
 	nodeid     *ecdsa.PrivateKey
-	topic      whisper.TopicType
+	topic      []byte
+	asymKeyID  string
 	filterID   string
 	symPass    string
 	msPassword string
@@ -72,27 +74,30 @@ var (
 
 // cmd arguments
 var (
-	echoMode       = flag.Bool("e", false, "echo mode: prints some arguments for diagnostics")
-	bootstrapMode  = flag.Bool("b", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
-	forwarderMode  = flag.Bool("f", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
-	mailServerMode = flag.Bool("s", false, "mail server mode: delivers expired messages on demand")
-	requestMail    = flag.Bool("r", false, "request expired messages from the bootstrap server")
-	asymmetricMode = flag.Bool("a", false, "use asymmetric encryption")
-	testMode       = flag.Bool("t", false, "use of predefined parameters for diagnostics")
-	generateKey    = flag.Bool("k", false, "generate and show the private key")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
+	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
+	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
+	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
+	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
 	argVerbosity = flag.Int("verbosity", int(log.LvlWarn), "log verbosity level")
 	argTTL       = flag.Uint("ttl", 30, "time-to-live for messages in seconds")
 	argWorkTime  = flag.Uint("work", 5, "work time in seconds")
-	argPoW       = flag.Float64("pow", whisper.MinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
-	argServerPoW = flag.Float64("mspow", whisper.MinimumPoW, "PoW requirement for Mail Server request")
+	argMaxSize   = flag.Int("maxsize", whisper.DefaultMaxMessageLength, "max size of message")
+	argPoW       = flag.Float64("pow", whisper.DefaultMinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argServerPoW = flag.Float64("mspow", whisper.DefaultMinimumPoW, "PoW requirement for Mail Server request")
 
-	argIP     = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:42786)")
-	argPub    = flag.String("pub", "", "public key for asymmetric encryption")
-	argDBPath = flag.String("dbpath", "", "path to the server's DB directory")
-	argIDFile = flag.String("idfile", "", "file name with node id (private key)")
-	argEnode  = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
-	argTopic  = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+	argIP      = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:42786)")
+	argPub     = flag.String("pub", "", "public key for asymmetric encryption")
+	argDBPath  = flag.String("dbpath", "", "path to the server's DB directory")
+	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
+	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
+	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4defaced)")
+	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
 )
 
 func main() {
@@ -124,7 +129,7 @@ func processArgs() {
 		if err != nil {
 			utils.Fatalf("Failed to parse the topic: %s", err)
 		}
-		topic = whisper.BytesToTopic(x)
+		topic = x
 	}
 
 	if *asymmetricMode && len(*argPub) > 0 {
@@ -134,6 +139,14 @@ func processArgs() {
 		}
 	}
 
+	if len(*argSaveDir) > 0 {
+		if _, err := os.Stat(*argSaveDir); os.IsNotExist(err) {
+			utils.Fatalf("Download directory '%s' does not exist", *argSaveDir)
+		}
+	} else if *fileExMode {
+		utils.Fatalf("Parameter 'savedir' is mandatory for file exchange mode")
+	}
+
 	if *echoMode {
 		echo()
 	}
@@ -199,9 +212,40 @@ func initialize() {
 		shh = whisper.New()
 	}
 
-	asymKey = shh.NewIdentity()
+	if *argPoW != whisper.DefaultMinimumPoW {
+		err := shh.SetMinimumPoW(*argPoW)
+		if err != nil {
+			utils.Fatalf("Failed to set PoW: %s", err)
+		}
+	}
+
+	if *argMaxSize != whisper.DefaultMaxMessageLength {
+		err := shh.SetMaxMessageLength(*argMaxSize)
+		if err != nil {
+			utils.Fatalf("Failed to set max message size: %s", err)
+		}
+	}
+
+	asymKeyID, err = shh.NewKeyPair()
+	if err != nil {
+		utils.Fatalf("Failed to generate a new key pair: %s", err)
+	}
+
+	asymKey, err = shh.GetPrivateKey(asymKeyID)
+	if err != nil {
+		utils.Fatalf("Failed to retrieve a new key pair: %s", err)
+	}
+
 	if nodeid == nil {
-		nodeid = shh.NewIdentity()
+		tmpID, err := shh.NewKeyPair()
+		if err != nil {
+			utils.Fatalf("Failed to generate a new key pair: %s", err)
+		}
+
+		nodeid, err = shh.GetPrivateKey(tmpID)
+		if err != nil {
+			utils.Fatalf("Failed to retrieve a new key pair: %s", err)
+		}
 	}
 
 	maxPeers := 80
@@ -213,7 +257,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("whisper-go", "5.0"),
+			Name:           common.MakeName("wnode", "5.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -288,8 +332,14 @@ func configureNode() {
 			}
 		}
 
-		shh.AddSymKey(symKeyName, []byte(symPass))
-		symKey = shh.GetSymKey(symKeyName)
+		symKeyID, err := shh.AddSymKeyFromPassword(symPass)
+		if err != nil {
+			utils.Fatalf("Failed to create symmetric key: %s", err)
+		}
+		symKey, err = shh.GetSymKey(symKeyID)
+		if err != nil {
+			utils.Fatalf("Failed to save symmetric key: %s", err)
+		}
 		if len(*argTopic) == 0 {
 			generateTopic([]byte(symPass))
 		}
@@ -302,12 +352,12 @@ func configureNode() {
 	}
 
 	filter := whisper.Filter{
-		KeySym:    symKey,
-		KeyAsym:   asymKey,
-		Topics:    []whisper.TopicType{topic},
-		AcceptP2P: p2pAccept,
+		KeySym:   symKey,
+		KeyAsym:  asymKey,
+		Topics:   [][]byte{topic},
+		AllowP2P: p2pAccept,
 	}
-	filterID, err = shh.Watch(&filter)
+	filterID, err = shh.Subscribe(&filter)
 	if err != nil {
 		utils.Fatalf("Failed to install filter: %s", err)
 	}
@@ -351,6 +401,8 @@ func run() {
 
 	if *requestMail {
 		requestExpiredMessagesLoop()
+	} else if *fileExMode {
+		sendFilesLoop()
 	} else {
 		sendLoop()
 	}
@@ -376,6 +428,31 @@ func sendLoop() {
 	}
 }
 
+func sendFilesLoop() {
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			close(done)
+			break
+		}
+		b, err := ioutil.ReadFile(s)
+		if err != nil {
+			fmt.Printf(">>> Error: %s \n", err)
+			continue
+		} else {
+			h := sendMsg(b)
+			if (h == common.Hash{}) {
+				fmt.Printf(">>> Error: message was not sent \n")
+			} else {
+				timestamp := time.Now().Unix()
+				from := crypto.PubkeyToAddress(asymKey.PublicKey)
+				fmt.Printf("\n%d <%x>: sent message with hash %x\n", timestamp, from, h)
+			}
+		}
+	}
+}
+
 func scanLine(prompt string) string {
 	if len(prompt) > 0 {
 		fmt.Print(prompt)
@@ -402,29 +479,36 @@ func scanUint(prompt string) uint32 {
 	return uint32(i)
 }
 
-func sendMsg(payload []byte) {
+func sendMsg(payload []byte) common.Hash {
 	params := whisper.MessageParams{
 		Src:      asymKey,
 		Dst:      pub,
 		KeySym:   symKey,
 		Payload:  payload,
-		Topic:    topic,
+		Topic:    whisper.BytesToTopic(topic),
 		TTL:      uint32(*argTTL),
 		PoW:      *argPoW,
 		WorkTime: uint32(*argWorkTime),
 	}
 
 	msg := whisper.NewSentMessage(&params)
+	if msg == nil {
+		fmt.Printf("failed to create new message (OS level error)")
+		os.Exit(0)
+	}
 	envelope, err := msg.Wrap(&params)
 	if err != nil {
 		fmt.Printf("failed to seal message: %v \n", err)
-		return
+		return common.Hash{}
 	}
 
 	err = shh.Send(envelope)
 	if err != nil {
 		fmt.Printf("failed to send message: %v \n", err)
+		return common.Hash{}
 	}
+
+	return envelope.Hash()
 }
 
 func messageLoop() {
@@ -440,7 +524,11 @@ func messageLoop() {
 		case <-ticker.C:
 			messages := f.Retrieve()
 			for _, msg := range messages {
-				printMessageInfo(msg)
+				if *fileExMode || len(msg.Payload) > 2048 {
+					writeMessageToFile(*argSaveDir, msg)
+				} else {
+					printMessageInfo(msg)
+				}
 			}
 		case <-done:
 			return
@@ -464,19 +552,47 @@ func printMessageInfo(msg *whisper.ReceivedMessage) {
 	}
 }
 
+func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
+	timestamp := fmt.Sprintf("%d", msg.Sent)
+	name := fmt.Sprintf("%x", msg.EnvelopeHash)
+
+	var address common.Address
+	if msg.Src != nil {
+		address = crypto.PubkeyToAddress(*msg.Src)
+	}
+
+	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+		// message from myself: don't save, only report
+		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
+	} else if len(dir) > 0 {
+		fullpath := filepath.Join(dir, name)
+		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		if err != nil {
+			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
+		} else {
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+		}
+	} else {
+		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+	}
+}
+
 func requestExpiredMessagesLoop() {
 	var key, peerID []byte
 	var timeLow, timeUpp uint32
 	var t string
 	var xt, empty whisper.TopicType
 
-	err := shh.AddSymKey(mailserver.MailServerKeyName, []byte(msPassword))
+	keyID, err := shh.AddSymKeyFromPassword(msPassword)
 	if err != nil {
 		utils.Fatalf("Failed to create symmetric key for mail request: %s", err)
 	}
-	key = shh.GetSymKey(mailserver.MailServerKeyName)
+	key, err = shh.GetSymKey(keyID)
+	if err != nil {
+		utils.Fatalf("Failed to save symmetric key for mail request: %s", err)
+	}
 	peerID = extractIdFromEnode(*argEnode)
-	shh.MarkPeerTrusted(peerID)
+	shh.AllowP2PMessagesFromPeer(peerID)
 
 	for {
 		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
@@ -509,6 +625,9 @@ func requestExpiredMessagesLoop() {
 		params.WorkTime = 5
 
 		msg := whisper.NewSentMessage(&params)
+		if msg == nil {
+			utils.Fatalf("failed to create new message (OS level error)")
+		}
 		env, err := msg.Wrap(&params)
 		if err != nil {
 			utils.Fatalf("Wrap failed: %s", err)
@@ -527,7 +646,6 @@ func extractIdFromEnode(s string) []byte {
 	n, err := discover.ParseNode(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)
-		return nil
 	}
 	return n.ID[:]
 }

common/compiler/solidity.go

@@ -25,20 +25,11 @@ import (
 	"io/ioutil"
 	"os/exec"
 	"regexp"
+	"strconv"
 	"strings"
-
-	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/crypto"
 )
 
-var (
-	versionRegexp = regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`)
-	solcParams    = []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
-		"--add-std",  // include standard lib contracts
-		"--optimize", // code optimizer switched on
-	}
-)
+var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
 type Contract struct {
 	Code string       `json:"code"`
@@ -54,17 +45,33 @@ type ContractInfo struct {
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
+	Metadata        string      `json:"metadata"`
 }
 
 // Solidity contains information about the solidity compiler.
 type Solidity struct {
 	Path, Version, FullVersion string
+	Major, Minor, Patch        int
 }
 
 // --combined-output format
 type solcOutput struct {
-	Contracts map[string]struct{ Bin, Abi, Devdoc, Userdoc string }
-	Version   string
+	Contracts map[string]struct {
+		Bin, Abi, Devdoc, Userdoc, Metadata string
+	}
+	Version string
+}
+
+func (s *Solidity) makeArgs() []string {
+	p := []string{
+		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--add-std",  // include standard lib contracts
+		"--optimize", // code optimizer switched on
+	}
+	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
+		p[1] += ",metadata"
+	}
+	return p
 }
 
 // SolidityVersion runs solc and parses its version output.
@@ -75,13 +82,23 @@ func SolidityVersion(solc string) (*Solidity, error) {
 	var out bytes.Buffer
 	cmd := exec.Command(solc, "--version")
 	cmd.Stdout = &out
-	if err := cmd.Run(); err != nil {
+	err := cmd.Run()
+	if err != nil {
 		return nil, err
 	}
-	s := &Solidity{
-		Path:        cmd.Path,
-		FullVersion: out.String(),
-		Version:     versionRegexp.FindString(out.String()),
+	matches := versionRegexp.FindStringSubmatch(out.String())
+	if len(matches) != 4 {
+		return nil, fmt.Errorf("can't parse solc version %q", out.String())
+	}
+	s := &Solidity{Path: cmd.Path, FullVersion: out.String(), Version: matches[0]}
+	if s.Major, err = strconv.Atoi(matches[1]); err != nil {
+		return nil, err
+	}
+	if s.Minor, err = strconv.Atoi(matches[2]); err != nil {
+		return nil, err
+	}
+	if s.Patch, err = strconv.Atoi(matches[3]); err != nil {
+		return nil, err
 	}
 	return s, nil
 }
@@ -91,13 +108,14 @@ func CompileSolidityString(solc, source string) (map[string]*Contract, error) {
 	if len(source) == 0 {
 		return nil, errors.New("solc: empty source string")
 	}
-	if solc == "" {
-		solc = "solc"
+	s, err := SolidityVersion(solc)
+	if err != nil {
+		return nil, err
 	}
-	args := append(solcParams, "--")
-	cmd := exec.Command(solc, append(args, "-")...)
+	args := append(s.makeArgs(), "--")
+	cmd := exec.Command(s.Path, append(args, "-")...)
 	cmd.Stdin = strings.NewReader(source)
-	return runsolc(cmd, source)
+	return s.run(cmd, source)
 }
 
 // CompileSolidity compiles all given Solidity source files.
@@ -109,15 +127,16 @@ func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract,
 	if err != nil {
 		return nil, err
 	}
-	if solc == "" {
-		solc = "solc"
+	s, err := SolidityVersion(solc)
+	if err != nil {
+		return nil, err
 	}
-	args := append(solcParams, "--")
-	cmd := exec.Command(solc, append(args, sourcefiles...)...)
-	return runsolc(cmd, source)
+	args := append(s.makeArgs(), "--")
+	cmd := exec.Command(s.Path, append(args, sourcefiles...)...)
+	return s.run(cmd, source)
 }
 
-func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
+func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 	var stderr, stdout bytes.Buffer
 	cmd.Stderr = &stderr
 	cmd.Stdout = &stdout
@@ -128,7 +147,6 @@ func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 	if err := json.Unmarshal(stdout.Bytes(), &output); err != nil {
 		return nil, err
 	}
-	shortVersion := versionRegexp.FindString(output.Version)
 
 	// Compilation succeeded, assemble and return the contracts.
 	contracts := make(map[string]*Contract)
@@ -151,12 +169,13 @@ func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
-				LanguageVersion: shortVersion,
-				CompilerVersion: shortVersion,
-				CompilerOptions: strings.Join(solcParams, " "),
+				LanguageVersion: s.Version,
+				CompilerVersion: s.Version,
+				CompilerOptions: strings.Join(s.makeArgs(), " "),
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,
+				Metadata:        info.Metadata,
 			},
 		}
 	}
@@ -174,13 +193,3 @@ func slurpFiles(files []string) (string, error) {
 	}
 	return concat.String(), nil
 }
-
-// SaveInfo serializes info to the given file and returns its Keccak256 hash.
-func SaveInfo(info *ContractInfo, filename string) (common.Hash, error) {
-	infojson, err := json.Marshal(info)
-	if err != nil {
-		return common.Hash{}, err
-	}
-	contenthash := common.BytesToHash(crypto.Keccak256(infojson))
-	return contenthash, ioutil.WriteFile(filename, infojson, 0600)
-}

common/compiler/solidity_test.go

@@ -17,14 +17,8 @@
 package compiler
 
 import (
-	"encoding/json"
-	"io/ioutil"
-	"os"
 	"os/exec"
-	"path"
 	"testing"
-
-	"github.com/expanse-org/go-expanse/common"
 )
 
 const (
@@ -36,7 +30,6 @@ contract test {
    }
 }
 `
-	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
 )
 
 func skipWithoutSolc(t *testing.T) {
@@ -57,7 +50,10 @@ func TestCompiler(t *testing.T) {
 	}
 	c, ok := contracts["test"]
 	if !ok {
-		t.Fatal("info for contract 'test' not present in result")
+		c, ok = contracts["<stdin>:test"]
+		if !ok {
+			t.Fatal("info for contract 'test' not present in result")
+		}
 	}
 	if c.Code == "" {
 		t.Error("empty code")
@@ -79,28 +75,3 @@ func TestCompileError(t *testing.T) {
 	}
 	t.Logf("error: %v", err)
 }
-
-func TestSaveInfo(t *testing.T) {
-	var cinfo ContractInfo
-	err := json.Unmarshal([]byte(testInfo), &cinfo)
-	if err != nil {
-		t.Errorf("%v", err)
-	}
-	filename := path.Join(os.TempDir(), "solctest.info.json")
-	os.Remove(filename)
-	cinfohash, err := SaveInfo(&cinfo, filename)
-	if err != nil {
-		t.Errorf("error extracting info: %v", err)
-	}
-	got, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Errorf("error reading '%v': %v", filename, err)
-	}
-	if string(got) != testInfo {
-		t.Errorf("incorrect info.json extracted, expected:\n%s\ngot\n%s", testInfo, string(got))
-	}
-	wantHash := common.HexToHash("0x22450a77f0c3ff7a395948d07bc1456881226a1b6325f4189cb5f1254a824080")
-	if cinfohash != wantHash {
-		t.Errorf("content hash for info is incorrect. expected %v, got %v", wantHash.Hex(), cinfohash.Hex())
-	}
-}

common/math/big.go

@@ -51,6 +51,9 @@ func (i *HexOrDecimal256) UnmarshalText(input []byte) error {
 
 // MarshalText implements encoding.TextMarshaler.
 func (i *HexOrDecimal256) MarshalText() ([]byte, error) {
+	if i == nil {
+		return []byte("0x0"), nil
+	}
 	return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil
 }
 

common/math/big_test.go

@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

consensus/clique/api.go

@@ -0,0 +1,119 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/rpc"
+)
+
+// API is a user facing RPC API to allow controlling the signer and voting
+// mechanisms of the proof-of-authority scheme.
+type API struct {
+	chain  consensus.ChainReader
+	clique *Clique
+}
+
+// GetSnapshot retrieves the state snapshot at a given block.
+func (api *API) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) {
+	// Retrieve the requested block number (or current if none requested)
+	var header *types.Header
+	if number == nil || *number == rpc.LatestBlockNumber {
+		header = api.chain.CurrentHeader()
+	} else {
+		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
+	}
+	// Ensure we have an actually valid block and return its snapshot
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+}
+
+// GetSnapshotAtHash retrieves the state snapshot at a given block.
+func (api *API) GetSnapshotAtHash(hash common.Hash) (*Snapshot, error) {
+	header := api.chain.GetHeaderByHash(hash)
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+}
+
+// GetSigners retrieves the list of authorized signers at the specified block.
+func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
+	// Retrieve the requested block number (or current if none requested)
+	var header *types.Header
+	if number == nil || *number == rpc.LatestBlockNumber {
+		header = api.chain.CurrentHeader()
+	} else {
+		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
+	}
+	// Ensure we have an actually valid block and return the signers from its snapshot
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return snap.signers(), nil
+}
+
+// GetSignersAtHash retrieves the state snapshot at a given block.
+func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
+	header := api.chain.GetHeaderByHash(hash)
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return snap.signers(), nil
+}
+
+// Proposals returns the current proposals the node tries to uphold and vote on.
+func (api *API) Proposals() map[common.Address]bool {
+	api.clique.lock.RLock()
+	defer api.clique.lock.RUnlock()
+
+	proposals := make(map[common.Address]bool)
+	for address, auth := range api.clique.proposals {
+		proposals[address] = auth
+	}
+	return proposals
+}
+
+// Propose injects a new authorization proposal that the signer will attempt to
+// push through.
+func (api *API) Propose(address common.Address, auth bool) {
+	api.clique.lock.Lock()
+	defer api.clique.lock.Unlock()
+
+	api.clique.proposals[address] = auth
+}
+
+// Discard drops a currently running proposal, stopping the signer from casting
+// further votes (either for or against).
+func (api *API) Discard(address common.Address) {
+	api.clique.lock.Lock()
+	defer api.clique.lock.Unlock()
+
+	delete(api.clique.proposals, address)
+}

consensus/clique/clique.go

@@ -0,0 +1,644 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package clique implements the proof-of-authority consensus engine.
+package clique
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/common/hexutil"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/crypto"
+	"github.com/expanse-org/go-expanse/crypto/sha3"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/expanse-org/go-expanse/rlp"
+	"github.com/expanse-org/go-expanse/rpc"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+const (
+	checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database
+	inmemorySnapshots  = 128  // Number of recent vote snapshots to keep in memory
+	inmemorySignatures = 1024 // Number of recent blocks to keep in memory
+
+	wiggleTime = 500 * time.Millisecond // Random delay (per signer) to allow concurrent signers
+)
+
+// Clique proof-of-authority protocol constants.
+var (
+	epochLength = uint64(30000) // Default number of blocks after which to checkpoint and reset the pending votes
+	blockPeriod = uint64(15)    // Default minimum difference between two consecutive block's timestamps
+
+	extraVanity = 32 // Fixed number of extra-data prefix bytes reserved for signer vanity
+	extraSeal   = 65 // Fixed number of extra-data suffix bytes reserved for signer seal
+
+	nonceAuthVote = hexutil.MustDecode("0xffffffffffffffff") // Magic nonce number to vote on adding a new signer
+	nonceDropVote = hexutil.MustDecode("0x0000000000000000") // Magic nonce number to vote on removing a signer.
+
+	uncleHash = types.CalcUncleHash(nil) // Always Keccak256(RLP([])) as uncles are meaningless outside of PoW.
+
+	diffInTurn = big.NewInt(2) // Block difficulty for in-turn signatures
+	diffNoTurn = big.NewInt(1) // Block difficulty for out-of-turn signatures
+)
+
+// Various error messages to mark blocks invalid. These should be private to
+// prevent engine specific errors from being referenced in the remainder of the
+// codebase, inherently breaking if the engine is swapped out. Please put common
+// error types into the consensus package.
+var (
+	// errUnknownBlock is returned when the list of signers is requested for a block
+	// that is not part of the local blockchain.
+	errUnknownBlock = errors.New("unknown block")
+
+	// errInvalidCheckpointBeneficiary is returned if a checkpoint/epoch transition
+	// block has a beneficiary set to non zeroes.
+	errInvalidCheckpointBeneficiary = errors.New("beneficiary in checkpoint block non-zero")
+
+	// errInvalidVote is returned if a nonce value is something else that the two
+	// allowed constants of 0x00..0 or 0xff..f.
+	errInvalidVote = errors.New("vote nonce not 0x00..0 or 0xff..f")
+
+	// errInvalidCheckpointVote is returned if a checkpoint/epoch transition block
+	// has a vote nonce set to non zeroes.
+	errInvalidCheckpointVote = errors.New("vote nonce in checkpoint block non-zero")
+
+	// errMissingVanity is returned if a block's extra-data section is shorter than
+	// 32 bytes, which is required to store the signer vanity.
+	errMissingVanity = errors.New("extra-data 32 byte vanity prefix missing")
+
+	// errMissingSignature is returned if a block's extra-data section doesn't seem
+	// to contain a 65 byte secp256k1 signature.
+	errMissingSignature = errors.New("extra-data 65 byte suffix signature missing")
+
+	// errExtraSigners is returned if non-checkpoint block contain signer data in
+	// their extra-data fields.
+	errExtraSigners = errors.New("non-checkpoint block contains extra signer list")
+
+	// drrInvalidCheckpointSigners is returned if a checkpoint block contains an
+	// invalid list of signers (i.e. non divisible by 20 bytes, or not the correct
+	// ones).
+	drrInvalidCheckpointSigners = errors.New("invalid signer list on checkpoint block")
+
+	// errInvalidMixDigest is returned if a block's mix digest is non zero.
+	errInvalidMixDigest = errors.New("non-zero mix digest")
+
+	// errInvalidUncleHash is returned if a block contains an non-empty uncle list.
+	errInvalidUncleHash = errors.New("non empty uncle hash")
+
+	// errInvalidDifficulty is returned if the difficulty of a block is not either
+	// of 1 or 2, or if the value does not match the turn of the signer.
+	errInvalidDifficulty = errors.New("invalid difficulty")
+
+	// ErrInvalidTimestamp is returned if the timestamp of a block is lower than
+	// the previous block's timestamp + the minimum block period.
+	ErrInvalidTimestamp = errors.New("invalid timestamp")
+
+	// errInvalidVotingChain is returned if an authorization list is attempted to
+	// be modified via out-of-range or non-contiguous headers.
+	errInvalidVotingChain = errors.New("invalid voting chain")
+
+	// errUnauthorized is returned if a header is signed by a non authorized entity.
+	errUnauthorized = errors.New("unauthorized")
+)
+
+// SignerFn is a signer callback function to request a hash to be signed by a
+// backing account.
+type SignerFn func(accounts.Account, []byte) ([]byte, error)
+
+// sigHash returns the hash which is used as input for the proof-of-authority
+// signing. It is the hash of the entire header apart from the 65 byte signature
+// contained at the end of the extra data.
+//
+// Note, the method requires the extra data to be at least 65 bytes, otherwise it
+// panics. This is done to avoid accidentally using both forms (signature present
+// or not), which could be abused to produce different hashes for the same header.
+func sigHash(header *types.Header) (hash common.Hash) {
+	hasher := sha3.NewKeccak256()
+
+	rlp.Encode(hasher, []interface{}{
+		header.ParentHash,
+		header.UncleHash,
+		header.Coinbase,
+		header.Root,
+		header.TxHash,
+		header.ReceiptHash,
+		header.Bloom,
+		header.Difficulty,
+		header.Number,
+		header.GasLimit,
+		header.GasUsed,
+		header.Time,
+		header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
+		header.MixDigest,
+		header.Nonce,
+	})
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+// ecrecover extracts the Ethereum account address from a signed header.
+func ecrecover(header *types.Header) (common.Address, error) {
+	// Retrieve the signature from the header extra-data
+	if len(header.Extra) < extraSeal {
+		return common.Address{}, errMissingSignature
+	}
+	signature := header.Extra[len(header.Extra)-extraSeal:]
+
+	// Recover the public key and the Ethereum address
+	pubkey, err := crypto.Ecrecover(sigHash(header).Bytes(), signature)
+	if err != nil {
+		return common.Address{}, err
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+
+	return signer, nil
+}
+
+// Clique is the proof-of-authority consensus engine proposed to support the
+// Ethereum testnet following the Ropsten attacks.
+type Clique struct {
+	config *params.CliqueConfig // Consensus engine configuration parameters
+	db     ethdb.Database       // Database to store and retrieve snapshot checkpoints
+
+	recents    *lru.ARCCache // Snapshots for recent block to speed up reorgs
+	signatures *lru.ARCCache // Signatures of recent blocks to speed up mining
+
+	proposals map[common.Address]bool // Current list of proposals we are pushing
+
+	signer common.Address // Ethereum address of the signing key
+	signFn SignerFn       // Signer function to authorize hashes with
+	lock   sync.RWMutex   // Protects the signer fields
+}
+
+// New creates a Clique proof-of-authority consensus engine with the initial
+// signers set to the ones provided by the user.
+func New(config *params.CliqueConfig, db ethdb.Database) *Clique {
+	// Set any missing consensus parameters to their defaults
+	conf := *config
+	if conf.Epoch == 0 {
+		conf.Epoch = epochLength
+	}
+	if conf.Period == 0 {
+		conf.Period = blockPeriod
+	}
+	// Allocate the snapshot caches and create the engine
+	recents, _ := lru.NewARC(inmemorySnapshots)
+	signatures, _ := lru.NewARC(inmemorySignatures)
+
+	return &Clique{
+		config:     &conf,
+		db:         db,
+		recents:    recents,
+		signatures: signatures,
+		proposals:  make(map[common.Address]bool),
+	}
+}
+
+// Author implements consensus.Engine, returning the Ethereum address recovered
+// from the signature in the header's extra-data section.
+func (c *Clique) Author(header *types.Header) (common.Address, error) {
+	return ecrecover(header)
+}
+
+// VerifyHeader checks whether a header conforms to the consensus rules.
+func (c *Clique) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error {
+	return c.verifyHeader(chain, header, nil)
+}
+
+// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers. The
+// method returns a quit channel to abort the operations and a results channel to
+// retrieve the async verifications (the order is that of the input slice).
+func (c *Clique) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
+	abort := make(chan struct{})
+	results := make(chan error, len(headers))
+
+	go func() {
+		for i, header := range headers {
+			err := c.verifyHeader(chain, header, headers[:i])
+
+			select {
+			case <-abort:
+				return
+			case results <- err:
+			}
+		}
+	}()
+	return abort, results
+}
+
+// verifyHeader checks whether a header conforms to the consensus rules.The
+// caller may optionally pass in a batch of parents (ascending order) to avoid
+// looking those up from the database. This is useful for concurrently verifying
+// a batch of new headers.
+func (c *Clique) verifyHeader(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	if header.Number == nil {
+		return errUnknownBlock
+	}
+	number := header.Number.Uint64()
+
+	// Don't waste time checking blocks from the future
+	if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+		return consensus.ErrFutureBlock
+	}
+	// Checkpoint blocks need to enforce zero beneficiary
+	checkpoint := (number % c.config.Epoch) == 0
+	if checkpoint && header.Coinbase != (common.Address{}) {
+		return errInvalidCheckpointBeneficiary
+	}
+	// Nonces must be 0x00..0 or 0xff..f, zeroes enforced on checkpoints
+	if !bytes.Equal(header.Nonce[:], nonceAuthVote) && !bytes.Equal(header.Nonce[:], nonceDropVote) {
+		return errInvalidVote
+	}
+	if checkpoint && !bytes.Equal(header.Nonce[:], nonceDropVote) {
+		return errInvalidCheckpointVote
+	}
+	// Check that the extra-data contains both the vanity and signature
+	if len(header.Extra) < extraVanity {
+		return errMissingVanity
+	}
+	if len(header.Extra) < extraVanity+extraSeal {
+		return errMissingSignature
+	}
+	// Ensure that the extra-data contains a signer list on checkpoint, but none otherwise
+	signersBytes := len(header.Extra) - extraVanity - extraSeal
+	if !checkpoint && signersBytes != 0 {
+		return errExtraSigners
+	}
+	if checkpoint && signersBytes%common.AddressLength != 0 {
+		return drrInvalidCheckpointSigners
+	}
+	// Ensure that the mix digest is zero as we don't have fork protection currently
+	if header.MixDigest != (common.Hash{}) {
+		return errInvalidMixDigest
+	}
+	// Ensure that the block doesn't contain any uncles which are meaningless in PoA
+	if header.UncleHash != uncleHash {
+		return errInvalidUncleHash
+	}
+	// Ensure that the block's difficulty is meaningful (may not be correct at this point)
+	if number > 0 {
+		if header.Difficulty == nil || (header.Difficulty.Cmp(diffInTurn) != 0 && header.Difficulty.Cmp(diffNoTurn) != 0) {
+			return errInvalidDifficulty
+		}
+	}
+	// All basic checks passed, verify cascading fields
+	return c.verifyCascadingFields(chain, header, parents)
+}
+
+// verifyCascadingFields verifies all the header fields that are not standalone,
+// rather depend on a batch of previous headers. The caller may optionally pass
+// in a batch of parents (ascending order) to avoid looking those up from the
+// database. This is useful for concurrently verifying a batch of new headers.
+func (c *Clique) verifyCascadingFields(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	// The genesis block is the always valid dead-end
+	number := header.Number.Uint64()
+	if number == 0 {
+		return nil
+	}
+	// Ensure that the block's timestamp isn't too close to it's parent
+	var parent *types.Header
+	if len(parents) > 0 {
+		parent = parents[len(parents)-1]
+	} else {
+		parent = chain.GetHeader(header.ParentHash, number-1)
+	}
+	if parent == nil || parent.Number.Uint64() != number-1 || parent.Hash() != header.ParentHash {
+		return consensus.ErrUnknownAncestor
+	}
+	if parent.Time.Uint64()+c.config.Period > header.Time.Uint64() {
+		return ErrInvalidTimestamp
+	}
+	// Retrieve the snapshot needed to verify this header and cache it
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, parents)
+	if err != nil {
+		return err
+	}
+	// If the block is a checkpoint block, verify the signer list
+	if number%c.config.Epoch == 0 {
+		signers := make([]byte, len(snap.Signers)*common.AddressLength)
+		for i, signer := range snap.signers() {
+			copy(signers[i*common.AddressLength:], signer[:])
+		}
+		extraSuffix := len(header.Extra) - extraSeal
+		if !bytes.Equal(header.Extra[extraVanity:extraSuffix], signers) {
+			return drrInvalidCheckpointSigners
+		}
+	}
+	// All basic checks passed, verify the seal and return
+	return c.verifySeal(chain, header, parents)
+}
+
+// snapshot retrieves the authorization snapshot at a given point in time.
+func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash common.Hash, parents []*types.Header) (*Snapshot, error) {
+	// Search for a snapshot in memory or on disk for checkpoints
+	var (
+		headers []*types.Header
+		snap    *Snapshot
+	)
+	for snap == nil {
+		// If an in-memory snapshot was found, use that
+		if s, ok := c.recents.Get(hash); ok {
+			snap = s.(*Snapshot)
+			break
+		}
+		// If an on-disk checkpoint snapshot can be found, use that
+		if number%checkpointInterval == 0 {
+			if s, err := loadSnapshot(c.config, c.db, hash); err == nil {
+				log.Trace("Loaded voting snapshot form disk", "number", number, "hash", hash)
+				snap = s
+				break
+			}
+		}
+		// If we're at block zero, make a snapshot
+		if number == 0 {
+			genesis := chain.GetHeaderByNumber(0)
+			if err := c.VerifyHeader(chain, genesis, false); err != nil {
+				return nil, err
+			}
+			signers := make([]common.Address, (len(genesis.Extra)-extraVanity-extraSeal)/common.AddressLength)
+			for i := 0; i < len(signers); i++ {
+				copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:])
+			}
+			snap = newSnapshot(c.config, 0, genesis.Hash(), signers)
+			if err := snap.store(c.db); err != nil {
+				return nil, err
+			}
+			log.Trace("Stored genesis voting snapshot to disk")
+			break
+		}
+		// No snapshot for this header, gather the header and move backward
+		var header *types.Header
+		if len(parents) > 0 {
+			// If we have explicit parents, pick from there (enforced)
+			header = parents[len(parents)-1]
+			if header.Hash() != hash || header.Number.Uint64() != number {
+				return nil, consensus.ErrUnknownAncestor
+			}
+			parents = parents[:len(parents)-1]
+		} else {
+			// No explicit parents (or no more left), reach out to the database
+			header = chain.GetHeader(hash, number)
+			if header == nil {
+				return nil, consensus.ErrUnknownAncestor
+			}
+		}
+		headers = append(headers, header)
+		number, hash = number-1, header.ParentHash
+	}
+	// Previous snapshot found, apply any pending headers on top of it
+	for i := 0; i < len(headers)/2; i++ {
+		headers[i], headers[len(headers)-1-i] = headers[len(headers)-1-i], headers[i]
+	}
+	snap, err := snap.apply(headers)
+	if err != nil {
+		return nil, err
+	}
+	c.recents.Add(snap.Hash, snap)
+
+	// If we've generated a new checkpoint snapshot, save to disk
+	if snap.Number%checkpointInterval == 0 && len(headers) > 0 {
+		if err = snap.store(c.db); err != nil {
+			return nil, err
+		}
+		log.Trace("Stored voting snapshot to disk", "number", snap.Number, "hash", snap.Hash)
+	}
+	return snap, err
+}
+
+// VerifyUncles implements consensus.Engine, always returning an error for any
+// uncles as this consensus mechanism doesn't permit uncles.
+func (c *Clique) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
+	if len(block.Uncles()) > 0 {
+		return errors.New("uncles not allowed")
+	}
+	return nil
+}
+
+// VerifySeal implements consensus.Engine, checking whether the signature contained
+// in the header satisfies the consensus protocol requirements.
+func (c *Clique) VerifySeal(chain consensus.ChainReader, header *types.Header) error {
+	return c.verifySeal(chain, header, nil)
+}
+
+// verifySeal checks whether the signature contained in the header satisfies the
+// consensus protocol requirements. The method accepts an optional list of parent
+// headers that aren't yet part of the local blockchain to generate the snapshots
+// from.
+func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	// Verifying the genesis block is not supported
+	number := header.Number.Uint64()
+	if number == 0 {
+		return errUnknownBlock
+	}
+	// Retrieve the snapshot needed to verify this header and cache it
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, parents)
+	if err != nil {
+		return err
+	}
+	c.recents.Add(snap.Hash, snap)
+
+	// Resolve the authorization key and check against signers
+	signer, err := ecrecover(header)
+	if err != nil {
+		return err
+	}
+	if _, ok := snap.Signers[signer]; !ok {
+		return errUnauthorized
+	}
+	for seen, recent := range snap.Recents {
+		if recent == signer {
+			// Signer is among recents, only fail if the current block doens't shift it out
+			if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit {
+				return errUnauthorized
+			}
+		}
+	}
+	// Ensure that the difficulty corresponts to the turn-ness of the signer
+	inturn := snap.inturn(header.Number.Uint64(), signer)
+	if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
+		return errInvalidDifficulty
+	}
+	if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
+		return errInvalidDifficulty
+	}
+	return nil
+}
+
+// Prepare implements consensus.Engine, preparing all the consensus fields of the
+// header for running the transactions on top.
+func (c *Clique) Prepare(chain consensus.ChainReader, header *types.Header) error {
+	// If the block isn't a checkpoint, cast a random vote (good enough fror now)
+	header.Coinbase = common.Address{}
+	header.Nonce = types.BlockNonce{}
+
+	number := header.Number.Uint64()
+	if number%c.config.Epoch != 0 {
+		c.lock.RLock()
+		if len(c.proposals) > 0 {
+			addresses := make([]common.Address, 0, len(c.proposals))
+			for address := range c.proposals {
+				addresses = append(addresses, address)
+			}
+			header.Coinbase = addresses[rand.Intn(len(addresses))]
+			if c.proposals[header.Coinbase] {
+				copy(header.Nonce[:], nonceAuthVote)
+			} else {
+				copy(header.Nonce[:], nonceDropVote)
+			}
+		}
+		c.lock.RUnlock()
+	}
+	// Assemble the voting snapshot and set the correct difficulty
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
+	if err != nil {
+		return err
+	}
+	header.Difficulty = diffNoTurn
+	if snap.inturn(header.Number.Uint64(), c.signer) {
+		header.Difficulty = diffInTurn
+	}
+	// Ensure the extra data has all it's components
+	if len(header.Extra) < extraVanity {
+		header.Extra = append(header.Extra, bytes.Repeat([]byte{0x00}, extraVanity-len(header.Extra))...)
+	}
+	header.Extra = header.Extra[:extraVanity]
+
+	if number%c.config.Epoch == 0 {
+		for _, signer := range snap.signers() {
+			header.Extra = append(header.Extra, signer[:]...)
+		}
+	}
+	header.Extra = append(header.Extra, make([]byte, extraSeal)...)
+
+	// Mix digest is reserved for now, set to empty
+	header.MixDigest = common.Hash{}
+
+	// Ensure the timestamp has the correct delay
+	parent := chain.GetHeader(header.ParentHash, number-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	header.Time = new(big.Int).Add(parent.Time, new(big.Int).SetUint64(c.config.Period))
+	if header.Time.Int64() < time.Now().Unix() {
+		header.Time = big.NewInt(time.Now().Unix())
+	}
+	return nil
+}
+
+// Finalize implements consensus.Engine, ensuring no uncles are set, nor block
+// rewards given, and returns the final block.
+func (c *Clique) Finalize(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
+	// No block rewards in PoA, so the state remains as is and uncles are dropped
+	header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
+	header.UncleHash = types.CalcUncleHash(nil)
+
+	// Assemble and return the final block for sealing
+	return types.NewBlock(header, txs, nil, receipts), nil
+}
+
+// Authorize injects a private key into the consensus engine to mint new blocks
+// with.
+func (c *Clique) Authorize(signer common.Address, signFn SignerFn) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.signer = signer
+	c.signFn = signFn
+}
+
+// Seal implements consensus.Engine, attempting to create a sealed block using
+// the local signing credentials.
+func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+	header := block.Header()
+
+	// Sealing the genesis block is not supported
+	number := header.Number.Uint64()
+	if number == 0 {
+		return nil, errUnknownBlock
+	}
+	// Don't hold the signer fields for the entire sealing procedure
+	c.lock.RLock()
+	signer, signFn := c.signer, c.signFn
+	c.lock.RUnlock()
+
+	// Bail out if we're unauthorized to sign a block
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
+	if err != nil {
+		return nil, err
+	}
+	if _, authorized := snap.Signers[signer]; !authorized {
+		return nil, errUnauthorized
+	}
+	// If we're amongs the recent signers, wait for the next block
+	for seen, recent := range snap.Recents {
+		if recent == signer {
+			// Signer is among recents, only wait if the current block doens't shift it out
+			if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit {
+				log.Info("Signed recently, must wait for others")
+				<-stop
+				return nil, nil
+			}
+		}
+	}
+	// Sweet, the protocol permits us to sign the block, wait for our time
+	delay := time.Unix(header.Time.Int64(), 0).Sub(time.Now())
+	if header.Difficulty.Cmp(diffNoTurn) == 0 {
+		// It's not our turn explicitly to sign, delay it a bit
+		wiggle := time.Duration(len(snap.Signers)/2+1) * wiggleTime
+		delay += time.Duration(rand.Int63n(int64(wiggle)))
+
+		log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
+	}
+	log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
+
+	select {
+	case <-stop:
+		return nil, nil
+	case <-time.After(delay):
+	}
+	// Sign all the things!
+	sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
+	if err != nil {
+		return nil, err
+	}
+	copy(header.Extra[len(header.Extra)-extraSeal:], sighash)
+
+	return block.WithSeal(header), nil
+}
+
+// APIs implements consensus.Engine, returning the user facing RPC API to allow
+// controlling the signer voting.
+func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API {
+	return []rpc.API{{
+		Namespace: "clique",
+		Version:   "1.0",
+		Service:   &API{chain: chain, clique: c},
+		Public:    false,
+	}}
+}

consensus/clique/snapshot.go

@@ -0,0 +1,299 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+// Vote represents a single vote that an authorized signer made to modify the
+// list of authorizations.
+type Vote struct {
+	Signer    common.Address `json:"signer"`    // Authorized signer that cast this vote
+	Block     uint64         `json:"block"`     // Block number the vote was cast in (expire old votes)
+	Address   common.Address `json:"address"`   // Account being voted on to change its authorization
+	Authorize bool           `json:"authorize"` // Whether to authorize or deauthorize the voted account
+}
+
+// Tally is a simple vote tally to keep the current score of votes. Votes that
+// go against the proposal aren't counted since it's equivalent to not voting.
+type Tally struct {
+	Authorize bool `json:"authorize"` // Whether the vote it about authorizing or kicking someone
+	Votes     int  `json:"votes"`     // Number of votes until now wanting to pass the proposal
+}
+
+// Snapshot is the state of the authorization voting at a given point in time.
+type Snapshot struct {
+	config *params.CliqueConfig // Consensus engine parameters to fine tune behavior
+
+	Number  uint64                      `json:"number"`  // Block number where the snapshot was created
+	Hash    common.Hash                 `json:"hash"`    // Block hash where the snapshot was created
+	Signers map[common.Address]struct{} `json:"signers"` // Set of authorized signers at this moment
+	Recents map[uint64]common.Address   `json:"recents"` // Set of recent signers for spam protections
+	Votes   []*Vote                     `json:"votes"`   // List of votes cast in chronological order
+	Tally   map[common.Address]Tally    `json:"tally"`   // Current vote tally to avoid recalculating
+}
+
+// newSnapshot create a new snapshot with the specified startup parameters. This
+// method does not initialize the set of recent signers, so only ever use if for
+// the genesis block.
+func newSnapshot(config *params.CliqueConfig, number uint64, hash common.Hash, signers []common.Address) *Snapshot {
+	snap := &Snapshot{
+		config:  config,
+		Number:  number,
+		Hash:    hash,
+		Signers: make(map[common.Address]struct{}),
+		Recents: make(map[uint64]common.Address),
+		Tally:   make(map[common.Address]Tally),
+	}
+	for _, signer := range signers {
+		snap.Signers[signer] = struct{}{}
+	}
+	return snap
+}
+
+// loadSnapshot loads an existing snapshot from the database.
+func loadSnapshot(config *params.CliqueConfig, db ethdb.Database, hash common.Hash) (*Snapshot, error) {
+	blob, err := db.Get(append([]byte("clique-"), hash[:]...))
+	if err != nil {
+		return nil, err
+	}
+	snap := new(Snapshot)
+	if err := json.Unmarshal(blob, snap); err != nil {
+		return nil, err
+	}
+	snap.config = config
+
+	return snap, nil
+}
+
+// store inserts the snapshot into the database.
+func (s *Snapshot) store(db ethdb.Database) error {
+	blob, err := json.Marshal(s)
+	if err != nil {
+		return err
+	}
+	return db.Put(append([]byte("clique-"), s.Hash[:]...), blob)
+}
+
+// copy creates a deep copy of the snapshot, though not the individual votes.
+func (s *Snapshot) copy() *Snapshot {
+	cpy := &Snapshot{
+		config:  s.config,
+		Number:  s.Number,
+		Hash:    s.Hash,
+		Signers: make(map[common.Address]struct{}),
+		Recents: make(map[uint64]common.Address),
+		Votes:   make([]*Vote, len(s.Votes)),
+		Tally:   make(map[common.Address]Tally),
+	}
+	for signer := range s.Signers {
+		cpy.Signers[signer] = struct{}{}
+	}
+	for block, signer := range s.Recents {
+		cpy.Recents[block] = signer
+	}
+	for address, tally := range s.Tally {
+		cpy.Tally[address] = tally
+	}
+	copy(cpy.Votes, s.Votes)
+
+	return cpy
+}
+
+// cast adds a new vote into the tally.
+func (s *Snapshot) cast(address common.Address, authorize bool) bool {
+	// Ensure the vote is meaningful
+	_, signer := s.Signers[address]
+	if (signer && authorize) || (!signer && !authorize) {
+		return false
+	}
+	// Cast the vote into an existing or new tally
+	if old, ok := s.Tally[address]; ok {
+		old.Votes++
+		s.Tally[address] = old
+	} else {
+		s.Tally[address] = Tally{Authorize: authorize, Votes: 1}
+	}
+	return true
+}
+
+// uncast removes a previously cast vote from the tally.
+func (s *Snapshot) uncast(address common.Address, authorize bool) bool {
+	// If there's no tally, it's a dangling vote, just drop
+	tally, ok := s.Tally[address]
+	if !ok {
+		return false
+	}
+	// Ensure we only revert counted votes
+	if tally.Authorize != authorize {
+		return false
+	}
+	// Otherwise revert the vote
+	if tally.Votes > 1 {
+		tally.Votes--
+		s.Tally[address] = tally
+	} else {
+		delete(s.Tally, address)
+	}
+	return true
+}
+
+// apply creates a new authorization snapshot by applying the given headers to
+// the original one.
+func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) {
+	// Allow passing in no headers for cleaner code
+	if len(headers) == 0 {
+		return s, nil
+	}
+	// Sanity check that the headers can be applied
+	for i := 0; i < len(headers)-1; i++ {
+		if headers[i+1].Number.Uint64() != headers[i].Number.Uint64()+1 {
+			return nil, errInvalidVotingChain
+		}
+	}
+	if headers[0].Number.Uint64() != s.Number+1 {
+		return nil, errInvalidVotingChain
+	}
+	// Iterate through the headers and create a new snapshot
+	snap := s.copy()
+
+	for _, header := range headers {
+		// Remove any votes on checkpoint blocks
+		number := header.Number.Uint64()
+		if number%s.config.Epoch == 0 {
+			snap.Votes = nil
+			snap.Tally = make(map[common.Address]Tally)
+		}
+		// Delete the oldest signer from the recent list to allow it signing again
+		if limit := uint64(len(snap.Signers)/2 + 1); number >= limit {
+			delete(snap.Recents, number-limit)
+		}
+		// Resolve the authorization key and check against signers
+		signer, err := ecrecover(header)
+		if err != nil {
+			return nil, err
+		}
+		if _, ok := snap.Signers[signer]; !ok {
+			return nil, errUnauthorized
+		}
+		for _, recent := range snap.Recents {
+			if recent == signer {
+				return nil, errUnauthorized
+			}
+		}
+		snap.Recents[number] = signer
+
+		// Header authorized, discard any previous votes from the signer
+		for i, vote := range snap.Votes {
+			if vote.Signer == signer && vote.Address == header.Coinbase {
+				// Uncast the vote from the cached tally
+				snap.uncast(vote.Address, vote.Authorize)
+
+				// Uncast the vote from the chronological list
+				snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+				break // only one vote allowed
+			}
+		}
+		// Tally up the new vote from the signer
+		var authorize bool
+		switch {
+		case bytes.Compare(header.Nonce[:], nonceAuthVote) == 0:
+			authorize = true
+		case bytes.Compare(header.Nonce[:], nonceDropVote) == 0:
+			authorize = false
+		default:
+			return nil, errInvalidVote
+		}
+		if snap.cast(header.Coinbase, authorize) {
+			snap.Votes = append(snap.Votes, &Vote{
+				Signer:    signer,
+				Block:     number,
+				Address:   header.Coinbase,
+				Authorize: authorize,
+			})
+		}
+		// If the vote passed, update the list of signers
+		if tally := snap.Tally[header.Coinbase]; tally.Votes > len(snap.Signers)/2 {
+			if tally.Authorize {
+				snap.Signers[header.Coinbase] = struct{}{}
+			} else {
+				delete(snap.Signers, header.Coinbase)
+
+				// Signer list shrunk, delete any leftover recent caches
+				if limit := uint64(len(snap.Signers)/2 + 1); number >= limit {
+					delete(snap.Recents, number-limit)
+				}
+				// Discard any previous votes the deauthorized signer cast
+				for i := 0; i < len(snap.Votes); i++ {
+					if snap.Votes[i].Signer == header.Coinbase {
+						// Uncast the vote from the cached tally
+						snap.uncast(snap.Votes[i].Address, snap.Votes[i].Authorize)
+
+						// Uncast the vote from the chronological list
+						snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+
+						i--
+					}
+				}
+			}
+			// Discard any previous votes around the just changed account
+			for i := 0; i < len(snap.Votes); i++ {
+				if snap.Votes[i].Address == header.Coinbase {
+					snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+					i--
+				}
+			}
+			delete(snap.Tally, header.Coinbase)
+		}
+	}
+	snap.Number += uint64(len(headers))
+	snap.Hash = headers[len(headers)-1].Hash()
+
+	return snap, nil
+}
+
+// signers retrieves the list of authorized signers in ascending order.
+func (s *Snapshot) signers() []common.Address {
+	signers := make([]common.Address, 0, len(s.Signers))
+	for signer := range s.Signers {
+		signers = append(signers, signer)
+	}
+	for i := 0; i < len(signers); i++ {
+		for j := i + 1; j < len(signers); j++ {
+			if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
+				signers[i], signers[j] = signers[j], signers[i]
+			}
+		}
+	}
+	return signers
+}
+
+// inturn returns if a signer at a given block height is in-turn or not.
+func (s *Snapshot) inturn(number uint64, signer common.Address) bool {
+	signers, offset := s.signers(), 0
+	for offset < len(signers) && signers[offset] != signer {
+		offset++
+	}
+	return (number % uint64(len(signers))) == uint64(offset)
+}

consensus/clique/snapshot_test.go

@@ -0,0 +1,405 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"math/big"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/crypto"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+type testerVote struct {
+	signer string
+	voted  string
+	auth   bool
+}
+
+// testerAccountPool is a pool to maintain currently active tester accounts,
+// mapped from textual names used in the tests below to actual Ethereum private
+// keys capable of signing transactions.
+type testerAccountPool struct {
+	accounts map[string]*ecdsa.PrivateKey
+}
+
+func newTesterAccountPool() *testerAccountPool {
+	return &testerAccountPool{
+		accounts: make(map[string]*ecdsa.PrivateKey),
+	}
+}
+
+func (ap *testerAccountPool) sign(header *types.Header, signer string) {
+	// Ensure we have a persistent key for the signer
+	if ap.accounts[signer] == nil {
+		ap.accounts[signer], _ = crypto.GenerateKey()
+	}
+	// Sign the header and embed the signature in extra data
+	sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
+	copy(header.Extra[len(header.Extra)-65:], sig)
+}
+
+func (ap *testerAccountPool) address(account string) common.Address {
+	// Ensure we have a persistent key for the account
+	if ap.accounts[account] == nil {
+		ap.accounts[account], _ = crypto.GenerateKey()
+	}
+	// Resolve and return the Ethereum address
+	return crypto.PubkeyToAddress(ap.accounts[account].PublicKey)
+}
+
+// testerChainReader implements consensus.ChainReader to access the genesis
+// block. All other methods and requests will panic.
+type testerChainReader struct {
+	db ethdb.Database
+}
+
+func (r *testerChainReader) Config() *params.ChainConfig                 { panic("not supported") }
+func (r *testerChainReader) CurrentHeader() *types.Header                { panic("not supported") }
+func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
+func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block   { panic("not supported") }
+func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header   { panic("not supported") }
+func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header {
+	if number == 0 {
+		return core.GetHeader(r.db, core.GetCanonicalHash(r.db, 0), 0)
+	}
+	panic("not supported")
+}
+
+// Tests that voting is evaluated correctly for various simple and complex scenarios.
+func TestVoting(t *testing.T) {
+	// Define the various voting scenarios to test
+	tests := []struct {
+		epoch   uint64
+		signers []string
+		votes   []testerVote
+		results []string
+	}{
+		{
+			// Single signer, no votes cast
+			signers: []string{"A"},
+			votes:   []testerVote{{signer: "A"}},
+			results: []string{"A"},
+		}, {
+			// Single signer, voting to add two others (only accept first, second needs 2 votes)
+			signers: []string{"A"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Two signers, voting to add three others (only accept first two, third needs 3 votes already)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B", voted: "C", auth: true},
+				{signer: "A", voted: "D", auth: true},
+				{signer: "B", voted: "D", auth: true},
+				{signer: "C"},
+				{signer: "A", voted: "E", auth: true},
+				{signer: "B", voted: "E", auth: true},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Single signer, dropping itself (weird, but one less cornercase by explicitly allowing this)
+			signers: []string{"A"},
+			votes: []testerVote{
+				{signer: "A", voted: "A", auth: false},
+			},
+			results: []string{},
+		}, {
+			// Two signers, actually needing mutual consent to drop either of them (not fulfilled)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Two signers, actually needing mutual consent to drop either of them (fulfilled)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B", voted: "B", auth: false},
+			},
+			results: []string{"A"},
+		}, {
+			// Three signers, two of them deciding to drop the third
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Four signers, consensus of two not being enough to drop anyone
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Four signers, consensus of three already being enough to drop someone
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Authorizations are counted once per signer per target
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Authorizing multiple accounts concurrently is permitted
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "D", auth: true},
+				{signer: "B"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: true},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Deauthorizations are counted once per signer per target
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B"},
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B"},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Deauthorizing multiple accounts concurrently is permitted
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Votes from deauthorized signers are discarded immediately (deauth votes)
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "C", voted: "B", auth: false},
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Votes from deauthorized signers are discarded immediately (auth votes)
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "C", voted: "B", auth: false},
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Cascading changes are not allowed, only the the account being voted on may change
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Changes reaching consensus out of bounds (via a deauth) execute on touch
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "C", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Changes reaching consensus out of bounds (via a deauth) may go out of consensus on first touch
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Ensure that pending votes don't survive authorization status changes. This
+			// corner case can only appear if a signer is quickly added, remove and then
+			// readded (or the inverse), while one of the original voters dropped. If a
+			// past vote is left cached in the system somewhere, this will interfere with
+			// the final signer outcome.
+			signers: []string{"A", "B", "C", "D", "E"},
+			votes: []testerVote{
+				{signer: "A", voted: "F", auth: true}, // Authorize F, 3 votes needed
+				{signer: "B", voted: "F", auth: true},
+				{signer: "C", voted: "F", auth: true},
+				{signer: "D", voted: "F", auth: false}, // Deauthorize F, 4 votes needed (leave A's previous vote "unchanged")
+				{signer: "E", voted: "F", auth: false},
+				{signer: "B", voted: "F", auth: false},
+				{signer: "C", voted: "F", auth: false},
+				{signer: "D", voted: "F", auth: true}, // Almost authorize F, 2/3 votes needed
+				{signer: "E", voted: "F", auth: true},
+				{signer: "B", voted: "A", auth: false}, // Deauthorize A, 3 votes needed
+				{signer: "C", voted: "A", auth: false},
+				{signer: "D", voted: "A", auth: false},
+				{signer: "B", voted: "F", auth: true}, // Finish authorizing F, 3/3 votes needed
+			},
+			results: []string{"B", "C", "D", "E", "F"},
+		}, {
+			// Epoch transitions reset all votes to allow chain checkpointing
+			epoch:   3,
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A"}, // Checkpoint block, (don't vote here, it's validated outside of snapshots)
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		},
+	}
+	// Run through the scenarios and test them
+	for i, tt := range tests {
+		// Create the account pool and generate the initial set of signers
+		accounts := newTesterAccountPool()
+
+		signers := make([]common.Address, len(tt.signers))
+		for j, signer := range tt.signers {
+			signers[j] = accounts.address(signer)
+		}
+		for j := 0; j < len(signers); j++ {
+			for k := j + 1; k < len(signers); k++ {
+				if bytes.Compare(signers[j][:], signers[k][:]) > 0 {
+					signers[j], signers[k] = signers[k], signers[j]
+				}
+			}
+		}
+		// Create the genesis block with the initial set of signers
+		genesis := &core.Genesis{
+			ExtraData: make([]byte, extraVanity+common.AddressLength*len(signers)+extraSeal),
+		}
+		for j, signer := range signers {
+			copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:])
+		}
+		// Create a pristine blockchain with the genesis injected
+		db, _ := ethdb.NewMemDatabase()
+		genesis.Commit(db)
+
+		// Assemble a chain of headers from the cast votes
+		headers := make([]*types.Header, len(tt.votes))
+		for j, vote := range tt.votes {
+			headers[j] = &types.Header{
+				Number:   big.NewInt(int64(j) + 1),
+				Time:     big.NewInt(int64(j) * int64(blockPeriod)),
+				Coinbase: accounts.address(vote.voted),
+				Extra:    make([]byte, extraVanity+extraSeal),
+			}
+			if j > 0 {
+				headers[j].ParentHash = headers[j-1].Hash()
+			}
+			if vote.auth {
+				copy(headers[j].Nonce[:], nonceAuthVote)
+			}
+			accounts.sign(headers[j], vote.signer)
+		}
+		// Pass all the headers through clique and ensure tallying succeeds
+		head := headers[len(headers)-1]
+
+		snap, err := New(&params.CliqueConfig{Epoch: tt.epoch}, db).snapshot(&testerChainReader{db: db}, head.Number.Uint64(), head.Hash(), headers)
+		if err != nil {
+			t.Errorf("test %d: failed to create voting snapshot: %v", i, err)
+			continue
+		}
+		// Verify the final list of signers against the expected ones
+		signers = make([]common.Address, len(tt.results))
+		for j, signer := range tt.results {
+			signers[j] = accounts.address(signer)
+		}
+		for j := 0; j < len(signers); j++ {
+			for k := j + 1; k < len(signers); k++ {
+				if bytes.Compare(signers[j][:], signers[k][:]) > 0 {
+					signers[j], signers[k] = signers[k], signers[j]
+				}
+			}
+		}
+		result := snap.signers()
+		if len(result) != len(signers) {
+			t.Errorf("test %d: signers mismatch: have %x, want %x", i, result, signers)
+			continue
+		}
+		for j := 0; j < len(result); j++ {
+			if !bytes.Equal(result[j][:], signers[j][:]) {
+				t.Errorf("test %d, signer %d: signer mismatch: have %x, want %x", i, j, result[j], signers[j])
+			}
+		}
+	}
+}

consensus/consensus.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package consensus implements different Ethereum consensus engines.
+package consensus
+
+import (
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/expanse-org/go-expanse/rpc"
+)
+
+// ChainReader defines a small collection of methods needed to access the local
+// blockchain during header and/or uncle verification.
+type ChainReader interface {
+	// Config retrieves the blockchain's chain configuration.
+	Config() *params.ChainConfig
+
+	// CurrentHeader retrieves the current header from the local chain.
+	CurrentHeader() *types.Header
+
+	// GetHeader retrieves a block header from the database by hash and number.
+	GetHeader(hash common.Hash, number uint64) *types.Header
+
+	// GetHeaderByNumber retrieves a block header from the database by number.
+	GetHeaderByNumber(number uint64) *types.Header
+
+	// GetHeaderByHash retrieves a block header from the database by its hash.
+	GetHeaderByHash(hash common.Hash) *types.Header
+
+	// GetBlock retrieves a block from the database by hash and number.
+	GetBlock(hash common.Hash, number uint64) *types.Block
+}
+
+// Engine is an algorithm agnostic consensus engine.
+type Engine interface {
+	// Author retrieves the Ethereum address of the account that minted the given
+	// block, which may be different from the header's coinbase if a consensus
+	// engine is based on signatures.
+	Author(header *types.Header) (common.Address, error)
+
+	// VerifyHeader checks whether a header conforms to the consensus rules of a
+	// given engine. Verifying the seal may be done optionally here, or explicitly
+	// via the VerifySeal method.
+	VerifyHeader(chain ChainReader, header *types.Header, seal bool) error
+
+	// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
+	// concurrently. The method returns a quit channel to abort the operations and
+	// a results channel to retrieve the async verifications (the order is that of
+	// the input slice).
+	VerifyHeaders(chain ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error)
+
+	// VerifyUncles verifies that the given block's uncles conform to the consensus
+	// rules of a given engine.
+	VerifyUncles(chain ChainReader, block *types.Block) error
+
+	// VerifySeal checks whether the crypto seal on a header is valid according to
+	// the consensus rules of the given engine.
+	VerifySeal(chain ChainReader, header *types.Header) error
+
+	// Prepare initializes the consensus fields of a block header according to the
+	// rules of a particular engine. The changes are executed inline.
+	Prepare(chain ChainReader, header *types.Header) error
+
+	// Finalize runs any post-transaction state modifications (e.g. block rewards)
+	// and assembles the final block.
+	//
+	// Note, the block header and state database might be updated to reflect any
+	// consensus rules that happen at finalization (e.g. block rewards).
+	Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction,
+		uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error)
+
+	// Seal generates a new block for the given input block with the local miner's
+	// seal place on top.
+	Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error)
+
+	// APIs returns the RPC APIs this consensus engine provides.
+	APIs(chain ChainReader) []rpc.API
+}
+
+// PoW is a consensus engine based on proof-of-work.
+type PoW interface {
+	Engine
+
+	// Hashrate returns the current mining hashrate of a PoW consensus engine.
+	Hashrate() float64
+}

consensus/errors.go

@@ -0,0 +1,33 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package consensus
+
+import "errors"
+
+var (
+	// ErrUnknownAncestor is returned when validating a block requires an ancestor
+	// that is unknown.
+	ErrUnknownAncestor = errors.New("unknown ancestor")
+
+	// ErrFutureBlock is returned when a block's timestamp is in the future according
+	// to the current node.
+	ErrFutureBlock = errors.New("block in the future")
+
+	// ErrInvalidNumber is returned if a block's number doesn't equal it's parent's
+	// plus one.
+	ErrInvalidNumber = errors.New("invalid block number")
+)

consensus/ethash/algorithm.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+package ethash
 
 import (
 	"encoding/binary"

consensus/ethash/algorithm_go1.7.go

@@ -16,7 +16,7 @@
 
 // +build !go1.8
 
-package pow
+package ethash
 
 // cacheSize calculates and returns the size of the ethash verification cache that
 // belongs to a certain block number. The cache size grows linearly, however, we

consensus/ethash/algorithm_go1.8.go

@@ -16,7 +16,7 @@
 
 // +build go1.8
 
-package pow
+package ethash
 
 import "math/big"
 

consensus/ethash/algorithm_go1.8_test.go

@@ -16,7 +16,7 @@
 
 // +build go1.8
 
-package pow
+package ethash
 
 import "testing"
 

consensus/ethash/algorithm_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+package ethash
 
 import (
 	"bytes"
@@ -704,8 +704,8 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 		go func(idx int) {
 			defer pend.Done()
 
-			ethash := NewFullEthash(cachedir, 0, 1, "", 0, 0)
-			if err := ethash.Verify(block); err != nil {
+			ethash := New(cachedir, 0, 1, "", 0, 0)
+			if err := ethash.VerifySeal(nil, block.Header()); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)
 			}
 		}(i)
@@ -713,17 +713,6 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 	pend.Wait()
 }
 
-func TestTestMode(t *testing.T) {
-	head := &types.Header{Difficulty: big.NewInt(100)}
-	ethash := NewTestEthash()
-	nonce, mix := ethash.Search(types.NewBlockWithHeader(head), nil)
-	head.Nonce = types.EncodeNonce(nonce)
-	copy(head.MixDigest[:], mix)
-	if err := ethash.Verify(types.NewBlockWithHeader(head)); err != nil {
-		t.Error("unexpected Verify error:", err)
-	}
-}
-
 // Benchmarks the cache generation performance.
 func BenchmarkCacheGeneration(b *testing.B) {
 	for i := 0; i < b.N; i++ {

consensus/ethash/consensus.go

@@ -0,0 +1,470 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/big"
+	"runtime"
+	"time"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/consensus/misc"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+	set "gopkg.in/fatih/set.v0"
+)
+
+// Ethash proof-of-work protocol constants.
+var (
+	blockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
+	maxUncles            = 2                 // Maximum number of uncles allowed in a single block
+)
+
+// Various error messages to mark blocks invalid. These should be private to
+// prevent engine specific errors from being referenced in the remainder of the
+// codebase, inherently breaking if the engine is swapped out. Please put common
+// error types into the consensus package.
+var (
+	errLargeBlockTime    = errors.New("timestamp too big")
+	errZeroBlockTime     = errors.New("timestamp equals parent's")
+	errTooManyUncles     = errors.New("too many uncles")
+	errDuplicateUncle    = errors.New("duplicate uncle")
+	errUncleIsAncestor   = errors.New("uncle is ancestor")
+	errDanglingUncle     = errors.New("uncle's parent is not ancestor")
+	errNonceOutOfRange   = errors.New("nonce out of range")
+	errInvalidDifficulty = errors.New("non-positive difficulty")
+	errInvalidMixDigest  = errors.New("invalid mix digest")
+	errInvalidPoW        = errors.New("invalid proof-of-work")
+)
+
+// Author implements consensus.Engine, returning the header's coinbase as the
+// proof-of-work verified author of the block.
+func (ethash *Ethash) Author(header *types.Header) (common.Address, error) {
+	return header.Coinbase, nil
+}
+
+// VerifyHeader checks whether a header conforms to the consensus rules of the
+// stock Ethereum ethash engine.
+func (ethash *Ethash) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull {
+		return nil
+	}
+	// Short circuit if the header is known, or it's parent not
+	number := header.Number.Uint64()
+	if chain.GetHeader(header.Hash(), number) != nil {
+		return nil
+	}
+	parent := chain.GetHeader(header.ParentHash, number-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	// Sanity checks passed, do a proper verification
+	return ethash.verifyHeader(chain, header, parent, false, seal)
+}
+
+// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
+// concurrently. The method returns a quit channel to abort the operations and
+// a results channel to retrieve the async verifications.
+func (ethash *Ethash) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull || len(headers) == 0 {
+		abort, results := make(chan struct{}), make(chan error, len(headers))
+		for i := 0; i < len(headers); i++ {
+			results <- nil
+		}
+		return abort, results
+	}
+
+	// Spawn as many workers as allowed threads
+	workers := runtime.GOMAXPROCS(0)
+	if len(headers) < workers {
+		workers = len(headers)
+	}
+
+	// Create a task channel and spawn the verifiers
+	var (
+		inputs = make(chan int)
+		done   = make(chan int, workers)
+		errors = make([]error, len(headers))
+		abort  = make(chan struct{})
+	)
+	for i := 0; i < workers; i++ {
+		go func() {
+			for index := range inputs {
+				errors[index] = ethash.verifyHeaderWorker(chain, headers, seals, index)
+				done <- index
+			}
+		}()
+	}
+
+	errorsOut := make(chan error, len(headers))
+	go func() {
+		defer close(inputs)
+		var (
+			in, out = 0, 0
+			checked = make([]bool, len(headers))
+			inputs  = inputs
+		)
+		for {
+			select {
+			case inputs <- in:
+				if in++; in == len(headers) {
+					// Reached end of headers. Stop sending to workers.
+					inputs = nil
+				}
+			case index := <-done:
+				for checked[index] = true; checked[out]; out++ {
+					errorsOut <- errors[out]
+					if out == len(headers)-1 {
+						return
+					}
+				}
+			case <-abort:
+				return
+			}
+		}
+	}()
+	return abort, errorsOut
+}
+
+func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainReader, headers []*types.Header, seals []bool, index int) error {
+	var parent *types.Header
+	if index == 0 {
+		parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1)
+	} else if headers[index-1].Hash() == headers[index].ParentHash {
+		parent = headers[index-1]
+	}
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	if chain.GetHeader(headers[index].Hash(), headers[index].Number.Uint64()) != nil {
+		return nil // known block
+	}
+	return ethash.verifyHeader(chain, headers[index], parent, false, seals[index])
+}
+
+// VerifyUncles verifies that the given block's uncles conform to the consensus
+// rules of the stock Ethereum ethash engine.
+func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull {
+		return nil
+	}
+	// Verify that there are at most 2 uncles included in this block
+	if len(block.Uncles()) > maxUncles {
+		return errTooManyUncles
+	}
+	// Gather the set of past uncles and ancestors
+	uncles, ancestors := set.New(), make(map[common.Hash]*types.Header)
+
+	number, parent := block.NumberU64()-1, block.ParentHash()
+	for i := 0; i < 7; i++ {
+		ancestor := chain.GetBlock(parent, number)
+		if ancestor == nil {
+			break
+		}
+		ancestors[ancestor.Hash()] = ancestor.Header()
+		for _, uncle := range ancestor.Uncles() {
+			uncles.Add(uncle.Hash())
+		}
+		parent, number = ancestor.ParentHash(), number-1
+	}
+	ancestors[block.Hash()] = block.Header()
+	uncles.Add(block.Hash())
+
+	// Verify each of the uncles that it's recent, but not an ancestor
+	for _, uncle := range block.Uncles() {
+		// Make sure every uncle is rewarded only once
+		hash := uncle.Hash()
+		if uncles.Has(hash) {
+			return errDuplicateUncle
+		}
+		uncles.Add(hash)
+
+		// Make sure the uncle has a valid ancestry
+		if ancestors[hash] != nil {
+			return errUncleIsAncestor
+		}
+		if ancestors[uncle.ParentHash] == nil || uncle.ParentHash == block.ParentHash() {
+			return errDanglingUncle
+		}
+		if err := ethash.verifyHeader(chain, uncle, ancestors[uncle.ParentHash], true, true); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// verifyHeader checks whether a header conforms to the consensus rules of the
+// stock Ethereum ethash engine.
+//
+// See YP section 4.3.4. "Block Header Validity"
+func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent *types.Header, uncle bool, seal bool) error {
+	// Ensure that the header's extra-data section is of a reasonable size
+	if uint64(len(header.Extra)) > params.MaximumExtraDataSize {
+		return fmt.Errorf("extra-data too long: %d > %d", len(header.Extra), params.MaximumExtraDataSize)
+	}
+	// Verify the header's timestamp
+	if uncle {
+		if header.Time.Cmp(math.MaxBig256) > 0 {
+			return errLargeBlockTime
+		}
+	} else {
+		if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+			return consensus.ErrFutureBlock
+		}
+	}
+	if header.Time.Cmp(parent.Time) <= 0 {
+		return errZeroBlockTime
+	}
+	// Verify the block's difficulty based in it's timestamp and parent's difficulty
+	expected := CalcDifficulty(chain.Config(), header.Time.Uint64(), parent.Time.Uint64(), parent.Number, parent.Difficulty)
+	if expected.Cmp(header.Difficulty) != 0 {
+		return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, expected)
+	}
+	// Verify that the gas limit remains within allowed bounds
+	diff := new(big.Int).Set(parent.GasLimit)
+	diff = diff.Sub(diff, header.GasLimit)
+	diff.Abs(diff)
+
+	limit := new(big.Int).Set(parent.GasLimit)
+	limit = limit.Div(limit, params.GasLimitBoundDivisor)
+
+	if diff.Cmp(limit) >= 0 || header.GasLimit.Cmp(params.MinGasLimit) < 0 {
+		return fmt.Errorf("invalid gas limit: have %v, want %v += %v", header.GasLimit, parent.GasLimit, limit)
+	}
+	// Verify that the block number is parent's +1
+	if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 {
+		return consensus.ErrInvalidNumber
+	}
+	// Verify the engine specific seal securing the block
+	if seal {
+		if err := ethash.VerifySeal(chain, header); err != nil {
+			return err
+		}
+	}
+	// If all checks passed, validate any special fields for hard forks
+	if err := misc.VerifyDAOHeaderExtraData(chain.Config(), header); err != nil {
+		return err
+	}
+	if err := misc.VerifyForkHashes(chain.Config(), header, uncle); err != nil {
+		return err
+	}
+	return nil
+}
+
+// CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty
+// that a new block should have when created at time given the parent block's time
+// and difficulty.
+//
+// TODO (karalabe): Move the chain maker into this package and make this private!
+func CalcDifficulty(config *params.ChainConfig, time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
+	if config.IsHomestead(new(big.Int).Add(parentNumber, common.Big1)) {
+		return calcDifficultyHomestead(time, parentTime, parentNumber, parentDiff)
+	}
+	return calcDifficultyFrontier(time, parentTime, parentNumber, parentDiff)
+}
+
+// Some weird constants to avoid constant memory allocs for them.
+var (
+	expDiffPeriod = big.NewInt(100000)
+	big10         = big.NewInt(10)
+	big60         = big.NewInt(60)
+	bigMinus99    = big.NewInt(-99)
+)
+
+// calcDifficultyHomestead is the difficulty adjustment algorithm. It returns
+// the difficulty that a new block should have when created at time given the
+// parent block's time and difficulty. The calculation uses the Homestead rules.
+func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.mediawiki
+	// algorithm:
+	// diff = (parent_diff +
+	//         (parent_diff / 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
+	//        ) + 2^(periodCount - 2)
+
+	bigTime := new(big.Int).SetUint64(time)
+	bigParentTime := new(big.Int).SetUint64(parentTime)
+
+	// holds intermediate values to make the algo easier to read & audit
+	x := new(big.Int)
+	y := new(big.Int)
+
+	// 1 - (block_timestamp -parent_timestamp) // 10
+	x.Sub(bigTime, bigParentTime)
+	x.Div(x, big60)
+	x.Sub(common.Big1, x)
+
+	// max(1 - (block_timestamp - parent_timestamp) // 10, -99)))
+	if x.Cmp(bigMinus99) < 0 {
+		x.Set(bigMinus99)
+	}
+
+	// (parent_diff + parent_diff // 512 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
+	y.Div(parentDiff, params.DifficultyBoundDivisor2)
+	x.Mul(y, x)
+	x.Add(parentDiff, x)
+
+	// minimum difficulty can ever be (before exponential factor)
+	if x.Cmp(params.MinimumDifficulty) < 0 {
+		x.Set(params.MinimumDifficulty)
+	}
+
+	return x
+}
+
+// calcDifficultyFrontier is the difficulty adjustment algorithm. It returns the
+// difficulty that a new block should have when created at time given the parent
+// block's time and difficulty. The calculation uses the Frontier rules.
+func calcDifficultyFrontier(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
+		diff := new(big.Int)
+		adjust := new(big.Int)
+
+		if parentNumber.Cmp(params.HardFork1) < 0 {
+			adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor)
+		} else {
+			adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor2)
+		}
+
+		bigTime := new(big.Int)
+		bigParentTime := new(big.Int)
+
+		bigTime.SetUint64(time)
+		bigParentTime.SetUint64(parentTime)
+
+		if bigTime.Sub(bigTime, bigParentTime).Cmp(params.DurationLimit) < 0 {
+			diff.Add(parentDiff, adjust)
+		} else {
+			diff.Sub(parentDiff, adjust)
+		}
+		if diff.Cmp(params.MinimumDifficulty) < 0 {
+			diff.Set(params.MinimumDifficulty)
+		}
+
+		periodCount := new(big.Int).Add(parentNumber, common.Big1)
+		periodCount.Div(periodCount, ExpDiffPeriod)
+		if periodCount.Cmp(common.Big1) > 0 {
+			// diff = diff + 2^(periodCount - 2)
+			expDiff := periodCount.Sub(periodCount, common.Big2)
+			expDiff.Exp(common.Big2, expDiff, nil)
+			diff.Add(diff, expDiff)
+			diff = math.BigMax(diff, params.MinimumDifficulty)
+		}
+
+		return diff
+}
+
+// VerifySeal implements consensus.Engine, checking whether the given block satisfies
+// the PoW difficulty requirements.
+func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Header) error {
+	// If we're running a fake PoW, accept any seal as valid
+	if ethash.fakeMode {
+		time.Sleep(ethash.fakeDelay)
+		if ethash.fakeFail == header.Number.Uint64() {
+			return errInvalidPoW
+		}
+		return nil
+	}
+	// If we're running a shared PoW, delegate verification to it
+	if ethash.shared != nil {
+		return ethash.shared.VerifySeal(chain, header)
+	}
+	// Sanity check that the block number is below the lookup table size (60M blocks)
+	number := header.Number.Uint64()
+	if number/epochLength >= uint64(len(cacheSizes)) {
+		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
+		return errNonceOutOfRange
+	}
+	// Ensure that we have a valid difficulty for the block
+	if header.Difficulty.Sign() <= 0 {
+		return errInvalidDifficulty
+	}
+	// Recompute the digest and PoW value and verify against the header
+	cache := ethash.cache(number)
+
+	size := datasetSize(number)
+	if ethash.tester {
+		size = 32 * 1024
+	}
+	digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+	if !bytes.Equal(header.MixDigest[:], digest) {
+		return errInvalidMixDigest
+	}
+	target := new(big.Int).Div(maxUint256, header.Difficulty)
+	if new(big.Int).SetBytes(result).Cmp(target) > 0 {
+		return errInvalidPoW
+	}
+	return nil
+}
+
+// Prepare implements consensus.Engine, initializing the difficulty field of a
+// header to conform to the ethash protocol. The changes are done inline.
+func (ethash *Ethash) Prepare(chain consensus.ChainReader, header *types.Header) error {
+	parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	header.Difficulty = CalcDifficulty(chain.Config(), header.Time.Uint64(),
+		parent.Time.Uint64(), parent.Number, parent.Difficulty)
+
+	return nil
+}
+
+// Finalize implements consensus.Engine, accumulating the block and uncle rewards,
+// setting the final state and assembling the block.
+func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
+	// Accumulate any block and uncle rewards and commit the final state root
+	AccumulateRewards(state, header, uncles)
+	header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
+
+	// Header seems complete, assemble into a block and return
+	return types.NewBlock(header, txs, uncles, receipts), nil
+}
+
+// Some weird constants to avoid constant memory allocs for them.
+var (
+	big8  = big.NewInt(8)
+	big32 = big.NewInt(32)
+)
+
+// AccumulateRewards credits the coinbase of the given block with the mining
+// reward. The total reward consists of the static block reward and rewards for
+// included uncles. The coinbase of each uncle block is also rewarded.
+//
+// TODO (karalabe): Move the chain maker into this package and make this private!
+func AccumulateRewards(state *state.StateDB, header *types.Header, uncles []*types.Header) {
+	reward := new(big.Int).Set(blockReward)
+	r := new(big.Int)
+	for _, uncle := range uncles {
+		r.Add(uncle.Number, big8)
+		r.Sub(r, header.Number)
+		r.Mul(r, blockReward)
+		r.Div(r, big8)
+		state.AddBalance(uncle.Coinbase, r)
+
+		r.Div(blockReward, big32)
+		reward.Add(reward, r)
+	}
+	state.AddBalance(header.Coinbase, reward)
+}

consensus/ethash/consensus_test.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"encoding/json"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+type diffTest struct {
+	ParentTimestamp    uint64
+	ParentDifficulty   *big.Int
+	CurrentTimestamp   uint64
+	CurrentBlocknumber *big.Int
+	CurrentDifficulty  *big.Int
+}
+
+func (d *diffTest) UnmarshalJSON(b []byte) (err error) {
+	var ext struct {
+		ParentTimestamp    string
+		ParentDifficulty   string
+		CurrentTimestamp   string
+		CurrentBlocknumber string
+		CurrentDifficulty  string
+	}
+	if err := json.Unmarshal(b, &ext); err != nil {
+		return err
+	}
+
+	d.ParentTimestamp = math.MustParseUint64(ext.ParentTimestamp)
+	d.ParentDifficulty = math.MustParseBig256(ext.ParentDifficulty)
+	d.CurrentTimestamp = math.MustParseUint64(ext.CurrentTimestamp)
+	d.CurrentBlocknumber = math.MustParseBig256(ext.CurrentBlocknumber)
+	d.CurrentDifficulty = math.MustParseBig256(ext.CurrentDifficulty)
+
+	return nil
+}
+
+func TestCalcDifficulty(t *testing.T) {
+	file, err := os.Open("../../tests/files/BasicTests/difficulty.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	tests := make(map[string]diffTest)
+	err = json.NewDecoder(file).Decode(&tests)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := &params.ChainConfig{HomesteadBlock: big.NewInt(1150000)}
+	for name, test := range tests {
+		number := new(big.Int).Sub(test.CurrentBlocknumber, big.NewInt(1))
+		diff := CalcDifficulty(config, test.CurrentTimestamp, test.ParentTimestamp, number, test.ParentDifficulty)
+		if diff.Cmp(test.CurrentDifficulty) != 0 {
+			t.Error(name, "failed. Expected", test.CurrentDifficulty, "and calculated", diff)
+		}
+	}
+}

consensus/ethash/ethash.go

@@ -14,10 +14,10 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+// Package ethash implements the ethash proof-of-work consensus engine.
+package ethash
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"math"
@@ -32,24 +32,21 @@ import (
 	"unsafe"
 
 	mmap "github.com/edsrzf/mmap-go"
+
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/rpc"
 	metrics "github.com/rcrowley/go-metrics"
 )
 
-var (
-	ErrInvalidDumpMagic  = errors.New("invalid dump magic")
-	ErrNonceOutOfRange   = errors.New("nonce out of range")
-	ErrInvalidDifficulty = errors.New("non-positive difficulty")
-	ErrInvalidMixDigest  = errors.New("invalid mix digest")
-	ErrInvalidPoW        = errors.New("pow difficulty invalid")
-)
+var ErrInvalidDumpMagic = errors.New("invalid dump magic")
 
 var (
 	// maxUint256 is a big integer representing 2^256-1
 	maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = NewFullEthash("", 3, 0, "", 1, 0)
+	sharedEthash = New("", 3, 0, "", 1, 0)
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -134,13 +131,16 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint
 	data := buffer[len(dumpMagic):]
 	generator(data)
 
-	if err := mem.Flush(); err != nil {
-		mem.Unmap()
-		dump.Close()
+	if err := mem.Unmap(); err != nil {
 		return nil, nil, nil, err
 	}
-	os.Rename(temp, path)
-	return dump, mem, data, nil
+	if err := dump.Close(); err != nil {
+		return nil, nil, nil, err
+	}
+	if err := os.Rename(temp, path); err != nil {
+		return nil, nil, nil, err
+	}
+	return memoryMap(path)
 }
 
 // cache wraps an ethash cache with some metadata to allow easier concurrent use.
@@ -321,7 +321,8 @@ func MakeDataset(block uint64, dir string) {
 	d.release()
 }
 
-// Ethash is a PoW data struture implementing the ethash algorithm.
+// Ethash is a consensus engine based on proot-of-work implementing the ethash
+// algorithm.
 type Ethash struct {
 	cachedir     string // Data directory to store the verification caches
 	cachesinmem  int    // Number of caches to keep in memory
@@ -334,15 +335,26 @@ type Ethash struct {
 	fcache   *cache              // Pre-generated cache for the estimated future epoch
 	datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often
 	fdataset *dataset            // Pre-generated dataset for the estimated future epoch
-	lock     sync.Mutex          // Ensures thread safety for the in-memory caches
 
+	// Mining related fields
+	rand     *rand.Rand    // Properly seeded random source for nonces
+	threads  int           // Number of threads to mine on if mining
+	update   chan struct{} // Notification channel to update mining parameters
 	hashrate metrics.Meter // Meter tracking the average hashrate
 
-	tester bool // Flag whether to use a smaller test dataset
+	// The fields below are hooks for testing
+	tester    bool          // Flag whether to use a smaller test dataset
+	shared    *Ethash       // Shared PoW verifier to avoid cache regeneration
+	fakeMode  bool          // Flag whether to disable PoW checking
+	fakeFull  bool          // Flag whether to disable all consensus rules
+	fakeFail  uint64        // Block number which fails PoW check even in fake mode
+	fakeDelay time.Duration // Time delay to sleep for before returning from verify
+
+	lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields
 }
 
-// NewFullEthash creates a full sized ethash PoW scheme.
-func NewFullEthash(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) PoW {
+// New creates a full sized ethash PoW scheme.
+func New(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) *Ethash {
 	if cachesinmem <= 0 {
 		log.Warn("One ethash cache must alwast be in memory", "requested", cachesinmem)
 		cachesinmem = 1
@@ -362,58 +374,55 @@ func NewFullEthash(cachedir string, cachesinmem, cachesondisk int, dagdir string
 		dagsondisk:   dagsondisk,
 		caches:       make(map[uint64]*cache),
 		datasets:     make(map[uint64]*dataset),
+		update:       make(chan struct{}),
 		hashrate:     metrics.NewMeter(),
 	}
 }
 
-// NewTestEthash creates a small sized ethash PoW scheme useful only for testing
+// NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
-func NewTestEthash() PoW {
+func NewTester() *Ethash {
 	return &Ethash{
 		cachesinmem: 1,
 		caches:      make(map[uint64]*cache),
 		datasets:    make(map[uint64]*dataset),
 		tester:      true,
+		update:      make(chan struct{}),
 		hashrate:    metrics.NewMeter(),
 	}
 }
 
-// NewSharedEthash creates a full sized ethash PoW shared between all requesters
-// running in the same process.
-func NewSharedEthash() PoW {
-	return sharedEthash
+// NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts
+// all blocks' seal as valid, though they still have to conform to the Ethereum
+// consensus rules.
+func NewFaker() *Ethash {
+	return &Ethash{fakeMode: true}
 }
 
-// Verify implements PoW, checking whether the given block satisfies the PoW
-// difficulty requirements.
-func (ethash *Ethash) Verify(block Block) error {
-	// Sanity check that the block number is below the lookup table size (60M blocks)
-	number := block.NumberU64()
-	if number/epochLength >= uint64(len(cacheSizes)) {
-		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
-		return ErrNonceOutOfRange
-	}
-	// Ensure that we have a valid difficulty for the block
-	difficulty := block.Difficulty()
-	if difficulty.Sign() <= 0 {
-		return ErrInvalidDifficulty
-	}
-	// Recompute the digest and PoW value and verify against the block
-	cache := ethash.cache(number)
+// NewFakeFailer creates a ethash consensus engine with a fake PoW scheme that
+// accepts all blocks as valid apart from the single one specified, though they
+// still have to conform to the Ethereum consensus rules.
+func NewFakeFailer(fail uint64) *Ethash {
+	return &Ethash{fakeMode: true, fakeFail: fail}
+}
 
-	size := datasetSize(number)
-	if ethash.tester {
-		size = 32 * 1024
-	}
-	digest, result := hashimotoLight(size, cache, block.HashNoNonce().Bytes(), block.Nonce())
-	if !bytes.Equal(block.MixDigest().Bytes(), digest) {
-		return ErrInvalidMixDigest
-	}
-	target := new(big.Int).Div(maxUint256, difficulty)
-	if new(big.Int).SetBytes(result).Cmp(target) > 0 {
-		return ErrInvalidPoW
-	}
-	return nil
+// NewFakeDelayer creates a ethash consensus engine with a fake PoW scheme that
+// accepts all blocks as valid, but delays verifications by some time, though
+// they still have to conform to the Ethereum consensus rules.
+func NewFakeDelayer(delay time.Duration) *Ethash {
+	return &Ethash{fakeMode: true, fakeDelay: delay}
+}
+
+// NewFullFaker creates a ethash consensus engine with a full fake scheme that
+// accepts all blocks as valid, without checking any consensus rules whatsoever.
+func NewFullFaker() *Ethash {
+	return &Ethash{fakeMode: true, fakeFull: true}
+}
+
+// NewShared creates a full sized ethash PoW shared between all requesters running
+// in the same process.
+func NewShared() *Ethash {
+	return &Ethash{shared: sharedEthash}
 }
 
 // cache tries to retrieve a verification cache for the specified block number
@@ -477,43 +486,6 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 	return current.cache
 }
 
-// Search implements PoW, attempting to find a nonce that satisfies the block's
-// difficulty requirements.
-func (ethash *Ethash) Search(block Block, stop <-chan struct{}) (uint64, []byte) {
-	var (
-		hash     = block.HashNoNonce().Bytes()
-		diff     = block.Difficulty()
-		target   = new(big.Int).Div(maxUint256, diff)
-		dataset  = ethash.dataset(block.NumberU64())
-		rand     = rand.New(rand.NewSource(time.Now().UnixNano()))
-		nonce    = uint64(rand.Int63())
-		attempts int64
-	)
-	// Start generating random nonces until we abort or find a good one
-	for {
-		select {
-		case <-stop:
-			// Mining terminated, update stats and abort
-			ethash.hashrate.Mark(attempts)
-			return 0, nil
-
-		default:
-			// We don't have to update hash rate on every nonce, so update after after 2^X nonces
-			attempts++
-			if (attempts % (1 << 15)) == 0 {
-				ethash.hashrate.Mark(attempts)
-				attempts = 0
-			}
-			// Compute the PoW value of this nonce
-			digest, result := hashimotoFull(dataset, hash, nonce)
-			if new(big.Int).SetBytes(result).Cmp(target) <= 0 {
-				return nonce, digest
-			}
-			nonce++
-		}
-	}
-}
-
 // dataset tries to retrieve a mining dataset for the specified block number
 // by first checking against a list of in-memory datasets, then against DAGs
 // stored on disk, and finally generating one if none can be found.
@@ -576,14 +548,51 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 	return current.dataset
 }
 
+// Threads returns the number of mining threads currently enabled. This doesn't
+// necessarily mean that mining is running!
+func (ethash *Ethash) Threads() int {
+	ethash.lock.Lock()
+	defer ethash.lock.Unlock()
+
+	return ethash.threads
+}
+
+// SetThreads updates the number of mining threads currently enabled. Calling
+// this method does not start mining, only sets the thread count. If zero is
+// specified, the miner will use all cores of the machine. Setting a thread
+// count below zero is allowed and will cause the miner to idle, without any
+// work being done.
+func (ethash *Ethash) SetThreads(threads int) {
+	ethash.lock.Lock()
+	defer ethash.lock.Unlock()
+
+	// If we're running a shared PoW, set the thread count on that instead
+	if ethash.shared != nil {
+		ethash.shared.SetThreads(threads)
+		return
+	}
+	// Update the threads and ping any running seal to pull in any changes
+	ethash.threads = threads
+	select {
+	case ethash.update <- struct{}{}:
+	default:
+	}
+}
+
 // Hashrate implements PoW, returning the measured rate of the search invocations
 // per second over the last minute.
 func (ethash *Ethash) Hashrate() float64 {
 	return ethash.hashrate.Rate1()
 }
 
-// EthashSeedHash is the seed to use for generating a vrification cache and the
-// mining dataset.
-func EthashSeedHash(block uint64) []byte {
+// APIs implements consensus.Engine, returning the user facing RPC APIs. Currently
+// that is empty.
+func (ethash *Ethash) APIs(chain consensus.ChainReader) []rpc.API {
+	return nil
+}
+
+// SeedHash is the seed to use for generating a verification cache and the mining
+// dataset.
+func SeedHash(block uint64) []byte {
 	return seedHash(block)
 }

consensus/ethash/ethash_test.go

@@ -0,0 +1,40 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/core/types"
+)
+
+// Tests that ethash works correctly in test mode.
+func TestTestMode(t *testing.T) {
+	head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+
+	ethash := NewTester()
+	block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil)
+	if err != nil {
+		t.Fatalf("failed to seal block: %v", err)
+	}
+	head.Nonce = types.EncodeNonce(block.Nonce())
+	head.MixDigest = block.MixDigest()
+	if err := ethash.VerifySeal(nil, head); err != nil {
+		t.Fatalf("unexpected verification error: %v", err)
+	}
+}

consensus/ethash/sealer.go

@@ -0,0 +1,149 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	crand "crypto/rand"
+	"math"
+	"math/big"
+	"math/rand"
+	"runtime"
+	"sync"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// Seal implements consensus.Engine, attempting to find a nonce that satisfies
+// the block's difficulty requirements.
+func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+	// If we're running a fake PoW, simply return a 0 nonce immediately
+	if ethash.fakeMode {
+		header := block.Header()
+		header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{}
+		return block.WithSeal(header), nil
+	}
+	// If we're running a shared PoW, delegate sealing to it
+	if ethash.shared != nil {
+		return ethash.shared.Seal(chain, block, stop)
+	}
+	// Create a runner and the multiple search threads it directs
+	abort := make(chan struct{})
+	found := make(chan *types.Block)
+
+	ethash.lock.Lock()
+	threads := ethash.threads
+	if ethash.rand == nil {
+		seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
+		if err != nil {
+			ethash.lock.Unlock()
+			return nil, err
+		}
+		ethash.rand = rand.New(rand.NewSource(seed.Int64()))
+	}
+	ethash.lock.Unlock()
+	if threads == 0 {
+		threads = runtime.NumCPU()
+	}
+	if threads < 0 {
+		threads = 0 // Allows disabling local mining without extra logic around local/remote
+	}
+	var pend sync.WaitGroup
+	for i := 0; i < threads; i++ {
+		pend.Add(1)
+		go func(id int, nonce uint64) {
+			defer pend.Done()
+			ethash.mine(block, id, nonce, abort, found)
+		}(i, uint64(ethash.rand.Int63()))
+	}
+	// Wait until sealing is terminated or a nonce is found
+	var result *types.Block
+	select {
+	case <-stop:
+		// Outside abort, stop all miner threads
+		close(abort)
+	case result = <-found:
+		// One of the threads found a block, abort all others
+		close(abort)
+	case <-ethash.update:
+		// Thread count was changed on user request, restart
+		close(abort)
+		pend.Wait()
+		return ethash.Seal(chain, block, stop)
+	}
+	// Wait for all miners to terminate and return the block
+	pend.Wait()
+	return result, nil
+}
+
+// mine is the actual proof-of-work miner that searches for a nonce starting from
+// seed that results in correct final block difficulty.
+func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) {
+	// Extract some data from the header
+	var (
+		header = block.Header()
+		hash   = header.HashNoNonce().Bytes()
+		target = new(big.Int).Div(maxUint256, header.Difficulty)
+
+		number  = header.Number.Uint64()
+		dataset = ethash.dataset(number)
+	)
+	// Start generating random nonces until we abort or find a good one
+	var (
+		attempts = int64(0)
+		nonce    = seed
+	)
+	logger := log.New("miner", id)
+	logger.Trace("Started ethash search for new nonces", "seed", seed)
+	for {
+		select {
+		case <-abort:
+			// Mining terminated, update stats and abort
+			logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed)
+			ethash.hashrate.Mark(attempts)
+			return
+
+		default:
+			// We don't have to update hash rate on every nonce, so update after after 2^X nonces
+			attempts++
+			if (attempts % (1 << 15)) == 0 {
+				ethash.hashrate.Mark(attempts)
+				attempts = 0
+			}
+			// Compute the PoW value of this nonce
+			digest, result := hashimotoFull(dataset, hash, nonce)
+			if new(big.Int).SetBytes(result).Cmp(target) <= 0 {
+				// Correct nonce found, create a new header with it
+				header = types.CopyHeader(header)
+				header.Nonce = types.EncodeNonce(nonce)
+				header.MixDigest = common.BytesToHash(digest)
+
+				// Seal and return a block (if still needed)
+				select {
+				case found <- block.WithSeal(header):
+					logger.Trace("Ethash nonce found and reported", "attempts", nonce-seed, "nonce", nonce)
+				case <-abort:
+					logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce)
+				}
+				return
+			}
+			nonce++
+		}
+	}
+}

consensus/ethash/xor.go

@@ -4,7 +4,7 @@
 
 // Source: https://golang.org/src/crypto/cipher/xor.go
 
-package pow
+package ethash
 
 import (
 	"runtime"

consensus/misc/dao.go

@@ -0,0 +1,85 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package misc
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+var (
+	// ErrBadProDAOExtra is returned if a header doens't support the DAO fork on a
+	// pro-fork client.
+	ErrBadProDAOExtra = errors.New("bad DAO pro-fork extra-data")
+
+	// ErrBadNoDAOExtra is returned if a header does support the DAO fork on a no-
+	// fork client.
+	ErrBadNoDAOExtra = errors.New("bad DAO no-fork extra-data")
+)
+
+// VerifyDAOHeaderExtraData validates the extra-data field of a block header to
+// ensure it conforms to DAO hard-fork rules.
+//
+// DAO hard-fork extension to the header validity:
+//   a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range
+//      with the fork specific extra-data set
+//   b) if the node is pro-fork, require blocks in the specific range to have the
+//      unique extra-data set.
+func VerifyDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error {
+	// Short circuit validation if the node doesn't care about the DAO fork
+	if config.DAOForkBlock == nil {
+		return nil
+	}
+	// Make sure the block is within the fork's modified extra-data range
+	limit := new(big.Int).Add(config.DAOForkBlock, params.DAOForkExtraRange)
+	if header.Number.Cmp(config.DAOForkBlock) < 0 || header.Number.Cmp(limit) >= 0 {
+		return nil
+	}
+	// Depending whether we support or oppose the fork, validate the extra-data contents
+	if config.DAOForkSupport {
+		if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
+			return ErrBadProDAOExtra
+		}
+	} else {
+		if bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
+			return ErrBadNoDAOExtra
+		}
+	}
+	// All ok, header has the same extra-data we expect
+	return nil
+}
+
+// ApplyDAOHardFork modifies the state database according to the DAO hard-fork
+// rules, transferring all balances of a set of DAO accounts to a single refund
+// contract.
+func ApplyDAOHardFork(statedb *state.StateDB) {
+	// Retrieve the contract to refund balances into
+	if !statedb.Exist(params.DAORefundContract) {
+		statedb.CreateAccount(params.DAORefundContract)
+	}
+
+	// Move every DAO account and extra-balance account funds into the refund contract
+	for _, addr := range params.DAODrainList() {
+		statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr))
+		statedb.SetBalance(addr, new(big.Int))
+	}
+}

consensus/misc/forks.go

@@ -0,0 +1,43 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package misc
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+// VerifyForkHashes verifies that blocks conforming to network hard-forks do have
+// the correct hashes, to avoid clients going off on different chains. This is an
+// optional feature.
+func VerifyForkHashes(config *params.ChainConfig, header *types.Header, uncle bool) error {
+	// We don't care about uncles
+	if uncle {
+		return nil
+	}
+	// If the homestead reprice hash is set, validate it
+	if config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 {
+		if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() {
+			return fmt.Errorf("homestead gas reprice fork: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash)
+		}
+	}
+	// All ok, return
+	return nil
+}

console/console_test.go

@@ -91,7 +91,7 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester {
 	}
 
 	// Create a networkless protocol stack and start an Ethereum service within
-	stack, err := node.New(&node.Config{DataDir: workspace, UseLightweightKDF: true, Name: testInstance, NoDiscovery: true})
+	stack, err := node.New(&node.Config{DataDir: workspace, UseLightweightKDF: true, Name: testInstance})
 	if err != nil {
 		t.Fatalf("failed to create node: %v", err)
 	}

core/bench_test.go

@@ -25,13 +25,13 @@ import (
 
 	"github.com/expanse-org/go-expanse/common"
 	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/crypto"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 func BenchmarkInsertChain_empty_memdb(b *testing.B) {
@@ -176,7 +176,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 	// Time the insertion of the new chain.
 	// State and blocks are stored in the same DB.
 	evmux := new(event.TypeMux)
-	chainman, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, evmux, vm.Config{})
+	chainman, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), evmux, vm.Config{})
 	defer chainman.Stop()
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -286,7 +286,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
-		chain, err := NewBlockChain(db, params.TestChainConfig, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+		chain, err := NewBlockChain(db, params.TestChainConfig, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 		if err != nil {
 			b.Fatalf("error creating chain: %v", err)
 		}

core/block_validator.go

@@ -19,23 +19,12 @@ package core
 import (
 	"fmt"
 	"math/big"
-	"time"
 
-	"github.com/expanse-org/go-expanse/common"
 	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
-	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
-	"gopkg.in/fatih/set.v0"
-)
-
-var (
-	ExpDiffPeriod = big.NewInt(100000)
-	big10         = big.NewInt(10)
-	big60         = big.NewInt(60)
-	bigMinus99    = big.NewInt(-99)
 )
 
 // BlockValidator is responsible for validating block headers, uncles and
@@ -45,66 +34,47 @@ var (
 type BlockValidator struct {
 	config *params.ChainConfig // Chain configuration options
 	bc     *BlockChain         // Canonical block chain
-	Pow    pow.PoW             // Proof of work used for validating
+	engine consensus.Engine    // Consensus engine used for validating
 }
 
 // NewBlockValidator returns a new block validator which is safe for re-use
-func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, pow pow.PoW) *BlockValidator {
+func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator {
 	validator := &BlockValidator{
 		config: config,
-		Pow:    pow,
+		engine: engine,
 		bc:     blockchain,
 	}
 	return validator
 }
 
-// ValidateBlock validates the given block's header and uncles and verifies the
-// the block header's transaction and uncle roots.
-//
-// ValidateBlock does not validate the header's pow. The pow work validated
-// separately so we can process them in parallel.
-//
-// ValidateBlock also validates and makes sure that any previous state (or present)
-// state that might or might not be present is checked to make sure that fast
-// sync has done it's job proper. This prevents the block validator from accepting
-// false positives where a header is present but the state is not.
-func (v *BlockValidator) ValidateBlock(block *types.Block) error {
+// ValidateBody validates the given block's uncles and verifies the the block
+// header's transaction and uncle roots. The headers are assumed to be already
+// validated at this point.
+func (v *BlockValidator) ValidateBody(block *types.Block) error {
+	// Check whether the block's known, and if not, that it's linkable
 	if v.bc.HasBlock(block.Hash()) {
 		if _, err := state.New(block.Root(), v.bc.chainDb); err == nil {
-			return &KnownBlockError{block.Number(), block.Hash()}
+			return ErrKnownBlock
 		}
 	}
 	parent := v.bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
 	if parent == nil {
-		return ParentError(block.ParentHash())
+		return consensus.ErrUnknownAncestor
 	}
 	if _, err := state.New(parent.Root(), v.bc.chainDb); err != nil {
-		return ParentError(block.ParentHash())
+		return consensus.ErrUnknownAncestor
 	}
-
+	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()
-	// validate the block header
-	if err := ValidateHeader(v.config, v.Pow, header, parent.Header(), false, false); err != nil {
+	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
 		return err
 	}
-	// verify the uncles are correctly rewarded
-	if err := v.VerifyUncles(block, parent); err != nil {
-		return err
+	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
+		return fmt.Errorf("uncle root hash mismatch: have %x, want %x", hash, header.UncleHash)
 	}
-
-	// Verify UncleHash before running other uncle validations
-	unclesSha := types.CalcUncleHash(block.Uncles())
-	if unclesSha != header.UncleHash {
-		return fmt.Errorf("invalid uncles root hash (remote: %x local: %x)", header.UncleHash, unclesSha)
+	if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
+		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
 	}
-
-	// The transactions Trie's root (R = (Tr [[i, RLP(T1)], [i, RLP(T2)], ... [n, RLP(Tn)]]))
-	// can be used by light clients to make sure they've received the correct Txs
-	txSha := types.DeriveSha(block.Transactions())
-	if txSha != header.TxHash {
-		return fmt.Errorf("invalid transaction root hash (remote: %x local: %x)", header.TxHash, txSha)
-	}
-
 	return nil
 }
 
@@ -112,10 +82,10 @@ func (v *BlockValidator) ValidateBlock(block *types.Block) error {
 // transition, such as amount of used gas, the receipt roots and the state root
 // itself. ValidateState returns a database batch if the validation was a success
 // otherwise nil and an error is returned.
-func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas *big.Int) (err error) {
+func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas *big.Int) error {
 	header := block.Header()
 	if block.GasUsed().Cmp(usedGas) != 0 {
-		return ValidationError(fmt.Sprintf("invalid gas used (remote: %v local: %v)", block.GasUsed(), usedGas))
+		return fmt.Errorf("invalid gas used (remote: %v local: %v)", block.GasUsed(), usedGas)
 	}
 	// Validate the received block's bloom with the one derived from the generated receipts.
 	// For valid blocks this should always validate to true.
@@ -136,217 +106,6 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat
 	return nil
 }
 
-// VerifyUncles verifies the given block's uncles and applies the Ethereum
-// consensus rules to the various block headers included; it will return an
-// error if any of the included uncle headers were invalid. It returns an error
-// if the validation failed.
-func (v *BlockValidator) VerifyUncles(block, parent *types.Block) error {
-	// validate that there are at most 2 uncles included in this block
-	if len(block.Uncles()) > 2 {
-		return ValidationError("Block can only contain maximum 2 uncles (contained %v)", len(block.Uncles()))
-	}
-
-	uncles := set.New()
-	ancestors := make(map[common.Hash]*types.Block)
-	for _, ancestor := range v.bc.GetBlocksFromHash(block.ParentHash(), 7) {
-		ancestors[ancestor.Hash()] = ancestor
-		// Include ancestors uncles in the uncle set. Uncles must be unique.
-		for _, uncle := range ancestor.Uncles() {
-			uncles.Add(uncle.Hash())
-		}
-	}
-	ancestors[block.Hash()] = block
-	uncles.Add(block.Hash())
-
-	for i, uncle := range block.Uncles() {
-		hash := uncle.Hash()
-		if uncles.Has(hash) {
-			// Error not unique
-			return UncleError("uncle[%d](%x) not unique", i, hash[:4])
-		}
-		uncles.Add(hash)
-
-		if ancestors[hash] != nil {
-			branch := fmt.Sprintf("  O - %x\n  |\n", block.Hash())
-			for h := range ancestors {
-				branch += fmt.Sprintf("  O - %x\n  |\n", h)
-			}
-			log.Warn(branch)
-			return UncleError("uncle[%d](%x) is ancestor", i, hash[:4])
-		}
-
-		if ancestors[uncle.ParentHash] == nil || uncle.ParentHash == parent.Hash() {
-			return UncleError("uncle[%d](%x)'s parent is not ancestor (%x)", i, hash[:4], uncle.ParentHash[0:4])
-		}
-
-		if err := ValidateHeader(v.config, v.Pow, uncle, ancestors[uncle.ParentHash].Header(), true, true); err != nil {
-			return ValidationError(fmt.Sprintf("uncle[%d](%x) header invalid: %v", i, hash[:4], err))
-		}
-	}
-
-	return nil
-}
-
-// ValidateHeader validates the given header and, depending on the pow arg,
-// checks the proof of work of the given header. Returns an error if the
-// validation failed.
-func (v *BlockValidator) ValidateHeader(header, parent *types.Header, checkPow bool) error {
-	// Short circuit if the parent is missing.
-	if parent == nil {
-		return ParentError(header.ParentHash)
-	}
-	// Short circuit if the header's already known or its parent is missing
-	if v.bc.HasHeader(header.Hash()) {
-		return nil
-	}
-	return ValidateHeader(v.config, v.Pow, header, parent, checkPow, false)
-}
-
-// Validates a header. Returns an error if the header is invalid.
-//
-// See YP section 4.3.4. "Block Header Validity"
-func ValidateHeader(config *params.ChainConfig, pow pow.PoW, header *types.Header, parent *types.Header, checkPow, uncle bool) error {
-	if uint64(len(header.Extra)) > params.MaximumExtraDataSize {
-		return fmt.Errorf("Header extra data too long (%d)", len(header.Extra))
-	}
-
-	if uncle {
-		if header.Time.Cmp(math.MaxBig256) == 1 {
-			return BlockTSTooBigErr
-		}
-	} else {
-		if header.Time.Cmp(big.NewInt(time.Now().Unix())) == 1 {
-			return BlockFutureErr
-		}
-	}
-	if header.Time.Cmp(parent.Time) != 1 {
-		return BlockEqualTSErr
-	}
-
-	expd := CalcDifficulty(config, header.Time.Uint64(), parent.Time.Uint64(), parent.Number, parent.Difficulty)
-	if expd.Cmp(header.Difficulty) != 0 {
-		return fmt.Errorf("Difficulty check failed for header (remote: %v local: %v)", header.Difficulty, expd)
-	}
-
-	a := new(big.Int).Set(parent.GasLimit)
-	a = a.Sub(a, header.GasLimit)
-	a.Abs(a)
-	b := new(big.Int).Set(parent.GasLimit)
-	b = b.Div(b, params.GasLimitBoundDivisor)
-	if !(a.Cmp(b) < 0) || (header.GasLimit.Cmp(params.MinGasLimit) == -1) {
-		return fmt.Errorf("GasLimit check failed for header (remote: %v local_max: %v)", header.GasLimit, b)
-	}
-
-	num := new(big.Int).Set(parent.Number)
-	num.Sub(header.Number, num)
-	if num.Cmp(big.NewInt(1)) != 0 {
-		return BlockNumberErr
-	}
-
-	if checkPow {
-		// Verify the nonce of the header. Return an error if it's not valid
-		if err := pow.Verify(types.NewBlockWithHeader(header)); err != nil {
-			return &BlockNonceErr{header.Number, header.Hash(), header.Nonce.Uint64()}
-		}
-	}
-	// If all checks passed, validate the extra-data field for hard forks
-	if err := ValidateDAOHeaderExtraData(config, header); err != nil {
-		return err
-	}
-	if !uncle && config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 {
-		if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() {
-			return ValidationError("Homestead gas reprice fork hash mismatch: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash)
-		}
-	}
-	return nil
-}
-
-// CalcDifficulty is the difficulty adjustment algorithm. It returns
-// the difficulty that a new block should have when created at time
-// given the parent block's time and difficulty.
-func CalcDifficulty(config *params.ChainConfig, time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
-	if config.IsHomestead(new(big.Int).Add(parentNumber, common.Big1)) {
-		return calcDifficultyHomestead(time, parentTime, parentNumber, parentDiff)
-	} else {
-		return calcDifficultyFrontier(time, parentTime, parentNumber, parentDiff)
-	}
-}
-
-func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
-	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.mediawiki
-	// algorithm:
-	// diff = (parent_diff +
-	//         (parent_diff / 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
-	//        ) + 2^(periodCount - 2)
-
-	bigTime := new(big.Int).SetUint64(time)
-	bigParentTime := new(big.Int).SetUint64(parentTime)
-
-	// holds intermediate values to make the algo easier to read & audit
-	x := new(big.Int)
-	y := new(big.Int)
-
-	// 1 - (block_timestamp -parent_timestamp) // 10
-	x.Sub(bigTime, bigParentTime)
-	x.Div(x, big60)
-	x.Sub(common.Big1, x)
-
-	// max(1 - (block_timestamp - parent_timestamp) // 10, -99)))
-	if x.Cmp(bigMinus99) < 0 {
-		x.Set(bigMinus99)
-	}
-
-	// (parent_diff + parent_diff // 512 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
-	y.Div(parentDiff, params.DifficultyBoundDivisor2)
-	x.Mul(y, x)
-	x.Add(parentDiff, x)
-
-	// minimum difficulty can ever be (before exponential factor)
-	if x.Cmp(params.MinimumDifficulty) < 0 {
-		x.Set(params.MinimumDifficulty)
-	}
-
-	return x
-}
-
-func calcDifficultyFrontier(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int {
-	diff := new(big.Int)
-	adjust := new(big.Int)
-
-	if parentNumber.Cmp(params.HardFork1) < 0 {
-		adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor)
-	} else {
-		adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor2)
-	}
-
-	bigTime := new(big.Int)
-	bigParentTime := new(big.Int)
-
-	bigTime.SetUint64(time)
-	bigParentTime.SetUint64(parentTime)
-
-	if bigTime.Sub(bigTime, bigParentTime).Cmp(params.DurationLimit) < 0 {
-		diff.Add(parentDiff, adjust)
-	} else {
-		diff.Sub(parentDiff, adjust)
-	}
-	if diff.Cmp(params.MinimumDifficulty) < 0 {
-		diff.Set(params.MinimumDifficulty)
-	}
-
-	periodCount := new(big.Int).Add(parentNumber, common.Big1)
-	periodCount.Div(periodCount, ExpDiffPeriod)
-	if periodCount.Cmp(common.Big1) > 0 {
-		// diff = diff + 2^(periodCount - 2)
-		expDiff := periodCount.Sub(periodCount, common.Big2)
-		expDiff.Exp(common.Big2, expDiff, nil)
-		diff.Add(diff, expDiff)
-		diff = math.BigMax(diff, params.MinimumDifficulty)
-	}
-
-	return diff
-}
-
 // CalcGasLimit computes the gas limit of the next block after parent.
 // The result may be modified by the caller.
 // This is miner strategy, not consensus protocol.

core/block_validator_test.go

@@ -17,64 +17,179 @@
 package core
 
 import (
-	"math/big"
+	"runtime"
 	"testing"
+	"time"
 
-	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
-func testGenesis(account common.Address, balance *big.Int) *Genesis {
-	return &Genesis{
-		Config: params.TestChainConfig,
-		Alloc:  GenesisAlloc{account: {Balance: balance}},
+// Tests that simple header verification works, for both good and bad blocks.
+func TestHeaderVerification(t *testing.T) {
+	// Create a simple chain to verify
+	var (
+		testdb, _ = ethdb.NewMemDatabase()
+		gspec     = &Genesis{Config: params.TestChainConfig}
+		genesis   = gspec.MustCommit(testdb)
+		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 8, nil)
+	)
+	headers := make([]*types.Header, len(blocks))
+	for i, block := range blocks {
+		headers[i] = block.Header()
+	}
+	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
+	chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
+
+	for i := 0; i < len(blocks); i++ {
+		for j, valid := range []bool{true, false} {
+			var results <-chan error
+
+			if valid {
+				engine := ethash.NewFaker()
+				_, results = engine.VerifyHeaders(chain, []*types.Header{headers[i]}, []bool{true})
+			} else {
+				engine := ethash.NewFakeFailer(headers[i].Number.Uint64())
+				_, results = engine.VerifyHeaders(chain, []*types.Header{headers[i]}, []bool{true})
+			}
+			// Wait for the verification result
+			select {
+			case result := <-results:
+				if (result == nil) != valid {
+					t.Errorf("test %d.%d: validity mismatch: have %v, want %v", i, j, result, valid)
+				}
+			case <-time.After(time.Second):
+				t.Fatalf("test %d.%d: verification timeout", i, j)
+			}
+			// Make sure no more data is returned
+			select {
+			case result := <-results:
+				t.Fatalf("test %d.%d: unexpected result returned: %v", i, j, result)
+			case <-time.After(25 * time.Millisecond):
+			}
+		}
+		chain.InsertChain(blocks[i : i+1])
 	}
 }
 
-func TestNumber(t *testing.T) {
-	chain := newTestBlockChain()
-	statedb, _ := state.New(chain.Genesis().Root(), chain.chainDb)
-	header := makeHeader(chain.config, chain.Genesis(), statedb)
-	header.Number = big.NewInt(3)
-	err := ValidateHeader(chain.config, pow.FakePow{}, header, chain.Genesis().Header(), false, false)
-	if err != BlockNumberErr {
-		t.Errorf("expected block number error, got %q", err)
-	}
+// Tests that concurrent header verification works, for both good and bad blocks.
+func TestHeaderConcurrentVerification2(t *testing.T)  { testHeaderConcurrentVerification(t, 2) }
+func TestHeaderConcurrentVerification8(t *testing.T)  { testHeaderConcurrentVerification(t, 8) }
+func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVerification(t, 32) }
 
-	header = makeHeader(chain.config, chain.Genesis(), statedb)
-	err = ValidateHeader(chain.config, pow.FakePow{}, header, chain.Genesis().Header(), false, false)
-	if err == BlockNumberErr {
-		t.Errorf("didn't expect block number error")
+func testHeaderConcurrentVerification(t *testing.T, threads int) {
+	// Create a simple chain to verify
+	var (
+		testdb, _ = ethdb.NewMemDatabase()
+		gspec     = &Genesis{Config: params.TestChainConfig}
+		genesis   = gspec.MustCommit(testdb)
+		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 8, nil)
+	)
+	headers := make([]*types.Header, len(blocks))
+	seals := make([]bool, len(blocks))
+
+	for i, block := range blocks {
+		headers[i] = block.Header()
+		seals[i] = true
+	}
+	// Set the number of threads to verify on
+	old := runtime.GOMAXPROCS(threads)
+	defer runtime.GOMAXPROCS(old)
+
+	// Run the header checker for the entire block chain at once both for a valid and
+	// also an invalid chain (enough if one arbitrary block is invalid).
+	for i, valid := range []bool{true, false} {
+		var results <-chan error
+
+		if valid {
+			chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
+			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
+		} else {
+			chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), new(event.TypeMux), vm.Config{})
+			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
+		}
+		// Wait for all the verification results
+		checks := make(map[int]error)
+		for j := 0; j < len(blocks); j++ {
+			select {
+			case result := <-results:
+				checks[j] = result
+
+			case <-time.After(time.Second):
+				t.Fatalf("test %d.%d: verification timeout", i, j)
+			}
+		}
+		// Check nonce check validity
+		for j := 0; j < len(blocks); j++ {
+			want := valid || (j < len(blocks)-2) // We chose the last-but-one nonce in the chain to fail
+			if (checks[j] == nil) != want {
+				t.Errorf("test %d.%d: validity mismatch: have %v, want %v", i, j, checks[j], want)
+			}
+			if !want {
+				// A few blocks after the first error may pass verification due to concurrent
+				// workers. We don't care about those in this test, just that the correct block
+				// errors out.
+				break
+			}
+		}
+		// Make sure no more data is returned
+		select {
+		case result := <-results:
+			t.Fatalf("test %d: unexpected result returned: %v", i, result)
+		case <-time.After(25 * time.Millisecond):
+		}
 	}
 }
 
-func TestPutReceipt(t *testing.T) {
-	db, _ := ethdb.NewMemDatabase()
+// Tests that aborting a header validation indeed prevents further checks from being
+// run, as well as checks that no left-over goroutines are leaked.
+func TestHeaderConcurrentAbortion2(t *testing.T)  { testHeaderConcurrentAbortion(t, 2) }
+func TestHeaderConcurrentAbortion8(t *testing.T)  { testHeaderConcurrentAbortion(t, 8) }
+func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion(t, 32) }
 
-	var addr common.Address
-	addr[0] = 1
-	var hash common.Hash
-	hash[0] = 2
+func testHeaderConcurrentAbortion(t *testing.T, threads int) {
+	// Create a simple chain to verify
+	var (
+		testdb, _ = ethdb.NewMemDatabase()
+		gspec     = &Genesis{Config: params.TestChainConfig}
+		genesis   = gspec.MustCommit(testdb)
+		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 1024, nil)
+	)
+	headers := make([]*types.Header, len(blocks))
+	seals := make([]bool, len(blocks))
 
-	receipt := new(types.Receipt)
-	receipt.Logs = []*types.Log{{
-		Address:     addr,
-		Topics:      []common.Hash{hash},
-		Data:        []byte("hi"),
-		BlockNumber: 42,
-		TxHash:      hash,
-		TxIndex:     0,
-		BlockHash:   hash,
-		Index:       0,
-	}}
+	for i, block := range blocks {
+		headers[i] = block.Header()
+		seals[i] = true
+	}
+	// Set the number of threads to verify on
+	old := runtime.GOMAXPROCS(threads)
+	defer runtime.GOMAXPROCS(old)
 
-	WriteReceipts(db, types.Receipts{receipt})
-	receipt = GetReceipt(db, common.Hash{})
-	if receipt == nil {
-		t.Error("expected to get 1 receipt, got none.")
+	// Start the verifications and immediately abort
+	chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), new(event.TypeMux), vm.Config{})
+	abort, results := chain.engine.VerifyHeaders(chain, headers, seals)
+	close(abort)
+
+	// Deplete the results channel
+	verified := 0
+	for depleted := false; !depleted; {
+		select {
+		case result := <-results:
+			if result != nil {
+				t.Errorf("header %d: validation failed: %v", verified, result)
+			}
+			verified++
+		case <-time.After(50 * time.Millisecond):
+			depleted = true
+		}
+	}
+	// Check that abortion was honored by not processing too many POWs
+	if verified > 2*threads {
+		t.Errorf("verification count too large: have %d, want below %d", verified, 2*threads)
 	}
 }

core/blockchain.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/expanse-org/go-expanse/common"
 	"github.com/expanse-org/go-expanse/common/mclock"
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
@@ -39,7 +40,6 @@ import (
 	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/metrics"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 	"github.com/expanse-org/go-expanse/rlp"
 	"github.com/expanse-org/go-expanse/trie"
 	"github.com/hashicorp/golang-lru"
@@ -104,7 +104,7 @@ type BlockChain struct {
 	procInterrupt int32          // interrupt signaler for block processing
 	wg            sync.WaitGroup // chain processing wait group for shutting down
 
-	pow       pow.PoW
+	engine    consensus.Engine
 	processor Processor // block processor interface
 	validator Validator // block and state validator interface
 	vmConfig  vm.Config
@@ -113,9 +113,9 @@ type BlockChain struct {
 }
 
 // NewBlockChain returns a fully initialised block chain using information
-// available in the database. It initialiser the default Ethereum Validator and
+// available in the database. It initialises the default Ethereum Validator and
 // Processor.
-func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.PoW, mux *event.TypeMux, vmConfig vm.Config) (*BlockChain, error) {
+func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, mux *event.TypeMux, vmConfig vm.Config) (*BlockChain, error) {
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
@@ -131,25 +131,22 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, pow pow.P
 		bodyRLPCache: bodyRLPCache,
 		blockCache:   blockCache,
 		futureBlocks: futureBlocks,
-		pow:          pow,
+		engine:       engine,
 		vmConfig:     vmConfig,
 		badBlocks:    badBlocks,
 	}
-	bc.SetValidator(NewBlockValidator(config, bc, pow))
-	bc.SetProcessor(NewStateProcessor(config, bc))
+	bc.SetValidator(NewBlockValidator(config, bc, engine))
+	bc.SetProcessor(NewStateProcessor(config, bc, engine))
 
-	gv := func() HeaderValidator { return bc.Validator() }
 	var err error
-	bc.hc, err = NewHeaderChain(chainDb, config, gv, bc.getProcInterrupt)
+	bc.hc, err = NewHeaderChain(chainDb, config, engine, bc.getProcInterrupt)
 	if err != nil {
 		return nil, err
 	}
-
 	bc.genesisBlock = bc.GetBlockByNumber(0)
 	if bc.genesisBlock == nil {
 		return nil, ErrNoGenesis
 	}
-
 	if err := bc.loadLastState(); err != nil {
 		return nil, err
 	}
@@ -233,9 +230,6 @@ func (self *BlockChain) loadLastState() error {
 	log.Info("Loaded most recent local full block", "number", self.currentBlock.Number(), "hash", self.currentBlock.Hash(), "td", blockTd)
 	log.Info("Loaded most recent local fast block", "number", self.currentFastBlock.Number(), "hash", self.currentFastBlock.Hash(), "td", fastTd)
 
-	// Try to be smart and issue a pow verification for the head to pre-generate caches
-	go self.pow.Verify(types.NewBlockWithHeader(currentHeader))
-
 	return nil
 }
 
@@ -383,9 +377,6 @@ func (self *BlockChain) Processor() Processor {
 	return self.processor
 }
 
-// AuxValidator returns the auxiliary validator (Proof of work atm)
-func (self *BlockChain) AuxValidator() pow.PoW { return self.pow }
-
 // State returns a new mutable state based on the current HEAD block.
 func (self *BlockChain) State() (*state.StateDB, error) {
 	return self.StateAt(self.CurrentBlock().Root())
@@ -840,7 +831,7 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err
 	// Calculate the total difficulty of the block
 	ptd := self.GetTd(block.ParentHash(), block.NumberU64()-1)
 	if ptd == nil {
-		return NonStatTy, ParentError(block.ParentHash())
+		return NonStatTy, consensus.ErrUnknownAncestor
 	}
 	// Make sure no inconsistent state is leaked during insertion
 	self.mu.Lock()
@@ -906,59 +897,57 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 		stats         = insertStats{startTime: mclock.Now()}
 		events        = make([]interface{}, 0, len(chain))
 		coalescedLogs []*types.Log
-		nonceChecked  = make([]bool, len(chain))
 	)
-
-	// Start the parallel nonce verifier.
-	nonceAbort, nonceResults := verifyNoncesFromBlocks(self.pow, chain)
-	defer close(nonceAbort)
+	// Start the parallel header verifier
+	headers := make([]*types.Header, len(chain))
+	seals := make([]bool, len(chain))
 
 	for i, block := range chain {
+		headers[i] = block.Header()
+		seals[i] = true
+	}
+	abort, results := self.engine.VerifyHeaders(self, headers, seals)
+	defer close(abort)
+
+	// Iterate over the blocks and insert when the verifier permits
+	for i, block := range chain {
+		// If the chain is terminating, stop processing blocks
 		if atomic.LoadInt32(&self.procInterrupt) == 1 {
 			log.Debug("Premature abort during blocks processing")
 			break
 		}
-		bstart := time.Now()
-		// Wait for block i's nonce to be verified before processing
-		// its state transition.
-		for !nonceChecked[i] {
-			r := <-nonceResults
-			nonceChecked[r.index] = true
-			if !r.valid {
-				invalid := chain[r.index]
-				return r.index, &BlockNonceErr{Hash: invalid.Hash(), Number: invalid.Number(), Nonce: invalid.Nonce()}
-			}
-		}
-
+		// If the header is a banned one, straight out abort
 		if BadHashes[block.Hash()] {
-			err := BadHashError(block.Hash())
-			self.reportBlock(block, nil, err)
-			return i, err
+			self.reportBlock(block, nil, ErrBlacklistedHash)
+			return i, ErrBlacklistedHash
+		}
+		// Wait for the block's verification to complete
+		bstart := time.Now()
+
+		err := <-results
+		if err == nil {
+			err = self.Validator().ValidateBody(block)
 		}
-		// Stage 1 validation of the block using the chain's validator
-		// interface.
-		err := self.Validator().ValidateBlock(block)
 		if err != nil {
-			if IsKnownBlockErr(err) {
+			if err == ErrKnownBlock {
 				stats.ignored++
 				continue
 			}
 
-			if err == BlockFutureErr {
+			if err == consensus.ErrFutureBlock {
 				// Allow up to MaxFuture second in the future blocks. If this limit
 				// is exceeded the chain is discarded and processed at a later time
 				// if given.
 				max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
-				if block.Time().Cmp(max) == 1 {
-					return i, fmt.Errorf("%v: BlockFutureErr, %v > %v", BlockFutureErr, block.Time(), max)
+				if block.Time().Cmp(max) > 0 {
+					return i, fmt.Errorf("future block: %v > %v", block.Time(), max)
 				}
-
 				self.futureBlocks.Add(block.Hash(), block)
 				stats.queued++
 				continue
 			}
 
-			if IsParentErr(err) && self.futureBlocks.Contains(block.ParentHash()) {
+			if err == consensus.ErrUnknownAncestor && self.futureBlocks.Contains(block.ParentHash()) {
 				self.futureBlocks.Add(block.Hash(), block)
 				stats.queued++
 				continue
@@ -1409,3 +1398,6 @@ func (self *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
 
 // Config retrieves the blockchain's chain configuration.
 func (self *BlockChain) Config() *params.ChainConfig { return self.config }
+
+// Engine retrieves the blockchain's consensus engine.
+func (self *BlockChain) Engine() consensus.Engine { return self.engine }

core/blockchain_test.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
@@ -31,18 +32,21 @@ import (
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 // newTestBlockChain creates a blockchain without validation.
-func newTestBlockChain() *BlockChain {
+func newTestBlockChain(fake bool) *BlockChain {
 	db, _ := ethdb.NewMemDatabase()
 	gspec := &Genesis{
 		Config:     params.TestChainConfig,
 		Difficulty: big.NewInt(1),
 	}
 	gspec.MustCommit(db)
-	blockchain, err := NewBlockChain(db, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	engine := ethash.NewFullFaker()
+	if !fake {
+		engine = ethash.NewTester()
+	}
+	blockchain, err := NewBlockChain(db, gspec.Config, engine, new(event.TypeMux), vm.Config{})
 	if err != nil {
 		panic(err)
 	}
@@ -117,9 +121,12 @@ func printChain(bc *BlockChain) {
 func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 	for _, block := range chain {
 		// Try and process the block
-		err := blockchain.Validator().ValidateBlock(block)
+		err := blockchain.engine.VerifyHeader(blockchain, block.Header(), true)
+		if err == nil {
+			err = blockchain.validator.ValidateBody(block)
+		}
 		if err != nil {
-			if IsKnownBlockErr(err) {
+			if err == ErrKnownBlock {
 				continue
 			}
 			return err
@@ -133,7 +140,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 			blockchain.reportBlock(block, receipts, err)
 			return err
 		}
-		err = blockchain.Validator().ValidateState(block, blockchain.GetBlockByHash(block.ParentHash()), statedb, receipts, usedGas)
+		err = blockchain.validator.ValidateState(block, blockchain.GetBlockByHash(block.ParentHash()), statedb, receipts, usedGas)
 		if err != nil {
 			blockchain.reportBlock(block, receipts, err)
 			return err
@@ -152,7 +159,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error {
 	for _, header := range chain {
 		// Try and validate the header
-		if err := blockchain.Validator().ValidateHeader(header, blockchain.GetHeaderByHash(header.ParentHash), false); err != nil {
+		if err := blockchain.engine.VerifyHeader(blockchain, header, false); err != nil {
 			return err
 		}
 		// Manually insert the header into the database, but don't reorganise (allows subsequent testing)
@@ -174,7 +181,7 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *
 }
 
 func TestLastBlock(t *testing.T) {
-	bchain := newTestBlockChain()
+	bchain := newTestBlockChain(false)
 	block := makeBlockChain(bchain.CurrentBlock(), 1, bchain.chainDb, 0)[0]
 	bchain.insert(block)
 	if block.Hash() != GetHeadBlockHash(bchain.chainDb) {
@@ -318,8 +325,7 @@ func testBrokenChain(t *testing.T, full bool) {
 
 type bproc struct{}
 
-func (bproc) ValidateBlock(*types.Block) error                        { return nil }
-func (bproc) ValidateHeader(*types.Header, *types.Header, bool) error { return nil }
+func (bproc) ValidateBody(*types.Block) error { return nil }
 func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas *big.Int) error {
 	return nil
 }
@@ -378,7 +384,7 @@ func testReorgShort(t *testing.T, full bool) {
 }
 
 func testReorg(t *testing.T, first, second []int, td int64, full bool) {
-	bc := newTestBlockChain()
+	bc := newTestBlockChain(true)
 
 	// Insert an easy and a difficult chain afterwards
 	if full {
@@ -422,7 +428,7 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) }
 func TestBadBlockHashes(t *testing.T)  { testBadHashes(t, true) }
 
 func testBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain()
+	bc := newTestBlockChain(true)
 
 	// Create a chain, ban a hash and try to import
 	var err error
@@ -435,8 +441,8 @@ func testBadHashes(t *testing.T, full bool) {
 		BadHashes[headers[2].Hash()] = true
 		_, err = bc.InsertHeaderChain(headers, 1)
 	}
-	if !IsBadHashError(err) {
-		t.Errorf("error mismatch: want: BadHashError, have: %v", err)
+	if err != ErrBlacklistedHash {
+		t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash)
 	}
 }
 
@@ -446,7 +452,7 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) }
 func TestReorgBadBlockHashes(t *testing.T)  { testReorgBadHashes(t, true) }
 
 func testReorgBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain()
+	bc := newTestBlockChain(true)
 
 	// Create a chain, import and ban afterwards
 	headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10)
@@ -473,7 +479,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 	}
 
 	// Create a new BlockChain and check that it rolled back the state.
-	ncm, err := NewBlockChain(bc.chainDb, bc.config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	ncm, err := NewBlockChain(bc.chainDb, bc.config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
@@ -504,47 +510,32 @@ func testInsertNonceError(t *testing.T, full bool) {
 		}
 		// Create and insert a chain with a failing nonce
 		var (
-			failAt   int
-			failRes  int
-			failNum  uint64
-			failHash common.Hash
+			failAt  int
+			failRes int
+			failNum uint64
 		)
 		if full {
 			blocks := makeBlockChain(blockchain.CurrentBlock(), i, db, 0)
 
 			failAt = rand.Int() % len(blocks)
 			failNum = blocks[failAt].NumberU64()
-			failHash = blocks[failAt].Hash()
-
-			blockchain.pow = failPow{failNum}
 
+			blockchain.engine = ethash.NewFakeFailer(failNum)
 			failRes, err = blockchain.InsertChain(blocks)
 		} else {
 			headers := makeHeaderChain(blockchain.CurrentHeader(), i, db, 0)
 
 			failAt = rand.Int() % len(headers)
 			failNum = headers[failAt].Number.Uint64()
-			failHash = headers[failAt].Hash()
-
-			blockchain.pow = failPow{failNum}
-			blockchain.validator = NewBlockValidator(params.TestChainConfig, blockchain, failPow{failNum})
 
+			blockchain.engine = ethash.NewFakeFailer(failNum)
+			blockchain.hc.engine = blockchain.engine
 			failRes, err = blockchain.InsertHeaderChain(headers, 1)
 		}
-		// Check that the returned error indicates the nonce failure.
+		// Check that the returned error indicates the failure.
 		if failRes != failAt {
 			t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
 		}
-		if !IsBlockNonceErr(err) {
-			t.Fatalf("test %d: error mismatch: have %v, want nonce error %T", i, err, err)
-		}
-		nerr := err.(*BlockNonceErr)
-		if nerr.Number.Uint64() != failNum {
-			t.Errorf("test %d: number mismatch: have %v, want %v", i, nerr.Number, failNum)
-		}
-		if nerr.Hash != failHash {
-			t.Errorf("test %d: hash mismatch: have %x, want %x", i, nerr.Hash[:4], failHash[:4])
-		}
 		// Check that all no blocks after the failing block have been inserted.
 		for j := 0; j < i-failAt; j++ {
 			if full {
@@ -569,9 +560,12 @@ func TestFastVsFullChains(t *testing.T) {
 		key, _   = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		address  = crypto.PubkeyToAddress(key.PublicKey)
 		funds    = big.NewInt(1000000000)
-		gspec    = testGenesis(address, funds)
-		genesis  = gspec.MustCommit(gendb)
-		signer   = types.NewEIP155Signer(gspec.Config.ChainId)
+		gspec    = &Genesis{
+			Config: params.TestChainConfig,
+			Alloc:  GenesisAlloc{address: {Balance: funds}},
+		}
+		genesis = gspec.MustCommit(gendb)
+		signer  = types.NewEIP155Signer(gspec.Config.ChainId)
 	)
 	blocks, receipts := GenerateChain(gspec.Config, genesis, gendb, 1024, func(i int, block *BlockGen) {
 		block.SetCoinbase(common.Address{0x00})
@@ -594,7 +588,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Import the chain as an archive node for the comparison baseline
 	archiveDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
-	archive, _ := NewBlockChain(archiveDb, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
@@ -603,7 +597,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Fast import the chain as a non-archive node to test
 	fastDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
 	headers := make([]*types.Header, len(blocks))
 	for i, block := range blocks {
@@ -680,8 +674,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	archiveDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
 
-	archive, _ := NewBlockChain(archiveDb, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
-
+	archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
 	}
@@ -692,7 +685,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a non-archive node and ensure all pointers are updated
 	fastDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
 	headers := make([]*types.Header, len(blocks))
 	for i, block := range blocks {
@@ -711,8 +704,8 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a light node and ensure all pointers are updated
 	lightDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(lightDb)
-	light, _ := NewBlockChain(lightDb, gspec.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
 
+	light, _ := NewBlockChain(lightDb, gspec.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	if n, err := light.InsertHeaderChain(headers, 1); err != nil {
 		t.Fatalf("failed to insert header %d: %v", n, err)
 	}
@@ -780,7 +773,7 @@ func TestChainTxReorgs(t *testing.T) {
 	})
 	// Import the chain. This runs all block validation rules.
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, evmux, vm.Config{})
+	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), evmux, vm.Config{})
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		t.Fatalf("failed to insert original chain[%d]: %v", i, err)
 	}
@@ -851,7 +844,7 @@ func TestLogReorgs(t *testing.T) {
 	)
 
 	var evmux event.TypeMux
-	blockchain, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, &evmux, vm.Config{})
+	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), &evmux, vm.Config{})
 
 	subs := evmux.Subscribe(RemovedLogsEvent{})
 	chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 2, func(i int, gen *BlockGen) {
@@ -883,13 +876,16 @@ func TestReorgSideEvent(t *testing.T) {
 		db, _   = ethdb.NewMemDatabase()
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
-		gspec   = testGenesis(addr1, big.NewInt(10000000000000))
+		gspec   = &Genesis{
+			Config: params.TestChainConfig,
+			Alloc:  GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}},
+		}
 		genesis = gspec.MustCommit(db)
 		signer  = types.NewEIP155Signer(gspec.Config.ChainId)
 	)
 
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, evmux, vm.Config{})
+	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), evmux, vm.Config{})
 
 	chain, _ := GenerateChain(gspec.Config, genesis, db, 3, func(i int, gen *BlockGen) {})
 	if _, err := blockchain.InsertChain(chain); err != nil {
@@ -959,7 +955,7 @@ done:
 
 // Tests if the canonical block can be fetched from the database during chain insertion.
 func TestCanonicalBlockRetrieval(t *testing.T) {
-	bc := newTestBlockChain()
+	bc := newTestBlockChain(false)
 	chain, _ := GenerateChain(bc.config, bc.genesisBlock, bc.chainDb, 10, func(i int, gen *BlockGen) {})
 
 	for i := range chain {
@@ -1004,7 +1000,7 @@ func TestEIP155Transition(t *testing.T) {
 		mux     event.TypeMux
 	)
 
-	blockchain, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, &mux, vm.Config{})
+	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), &mux, vm.Config{})
 	blocks, _ := GenerateChain(gspec.Config, genesis, db, 4, func(i int, block *BlockGen) {
 		var (
 			tx      *types.Transaction
@@ -1110,7 +1106,7 @@ func TestEIP161AccountRemoval(t *testing.T) {
 		}
 		genesis       = gspec.MustCommit(db)
 		mux           event.TypeMux
-		blockchain, _ = NewBlockChain(db, gspec.Config, pow.FakePow{}, &mux, vm.Config{})
+		blockchain, _ = NewBlockChain(db, gspec.Config, ethash.NewFaker(), &mux, vm.Config{})
 	)
 	blocks, _ := GenerateChain(gspec.Config, genesis, db, 3, func(i int, block *BlockGen) {
 		var (

core/chain_makers.go

@@ -21,13 +21,14 @@ import (
 	"math/big"
 
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
+	"github.com/expanse-org/go-expanse/consensus/misc"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 // So we can deterministically seed different blockchains
@@ -84,7 +85,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
 		b.SetCoinbase(common.Address{})
 	}
 	b.statedb.StartRecord(tx.Hash(), common.Hash{}, len(b.txs))
-	receipt, _, err := ApplyTransaction(b.config, nil, b.gasPool, b.statedb, b.header, tx, b.header.GasUsed, vm.Config{})
+	receipt, _, err := ApplyTransaction(b.config, nil, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, b.header.GasUsed, vm.Config{})
 	if err != nil {
 		panic(err)
 	}
@@ -141,7 +142,7 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = CalcDifficulty(b.config, b.header.Time.Uint64(), b.parent.Time().Uint64(), b.parent.Number(), b.parent.Difficulty())
+	b.header.Difficulty = ethash.CalcDifficulty(b.config, b.header.Time.Uint64(), b.parent.Time().Uint64(), b.parent.Number(), b.parent.Difficulty())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -173,13 +174,13 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, db ethdb.Dat
 			}
 		}
 		if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(h.Number) == 0 {
-			ApplyDAOHardFork(statedb)
+			misc.ApplyDAOHardFork(statedb)
 		}
 		// Execute any user modifications to the block and finalize it
 		if gen != nil {
 			gen(i, b)
 		}
-		AccumulateRewards(statedb, h, b.uncles)
+		ethash.AccumulateRewards(statedb, h, b.uncles)
 		root, err := statedb.Commit(config.IsEIP158(h.Number))
 		if err != nil {
 			panic(fmt.Sprintf("state write error: %v", err))
@@ -212,7 +213,7 @@ func makeHeader(config *params.ChainConfig, parent *types.Block, state *state.St
 		Root:       state.IntermediateRoot(config.IsEIP158(parent.Number())),
 		ParentHash: parent.Hash(),
 		Coinbase:   parent.Coinbase(),
-		Difficulty: CalcDifficulty(config, time.Uint64(), new(big.Int).Sub(time, big.NewInt(10)).Uint64(), parent.Number(), parent.Difficulty()),
+		Difficulty: ethash.CalcDifficulty(config, time.Uint64(), new(big.Int).Sub(time, big.NewInt(10)).Uint64(), parent.Number(), parent.Difficulty()),
 		GasLimit:   CalcGasLimit(parent),
 		GasUsed:    new(big.Int),
 		Number:     new(big.Int).Add(parent.Number(), common.Big1),
@@ -229,7 +230,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) {
 	db, _ := ethdb.NewMemDatabase()
 	genesis := gspec.MustCommit(db)
 
-	blockchain, _ := NewBlockChain(db, params.AllProtocolChanges, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+	blockchain, _ := NewBlockChain(db, params.AllProtocolChanges, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil

core/chain_makers_test.go

@@ -20,13 +20,13 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/crypto"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 func ExampleGenerateChain() {
@@ -81,7 +81,7 @@ func ExampleGenerateChain() {
 
 	// Import the chain. This runs all block validation rules.
 	evmux := &event.TypeMux{}
-	blockchain, _ := NewBlockChain(db, gspec.Config, pow.FakePow{}, evmux, vm.Config{})
+	blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), evmux, vm.Config{})
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err)
 		return

core/chain_pow.go

@@ -1,87 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"runtime"
-
-	"github.com/expanse-org/go-expanse/core/types"
-	"github.com/expanse-org/go-expanse/pow"
-)
-
-// nonceCheckResult contains the result of a nonce verification.
-type nonceCheckResult struct {
-	index int  // Index of the item verified from an input array
-	valid bool // Result of the nonce verification
-}
-
-// verifyNoncesFromHeaders starts a concurrent header nonce verification,
-// returning a quit channel to abort the operations and a results channel
-// to retrieve the async verifications.
-func verifyNoncesFromHeaders(checker pow.PoW, headers []*types.Header) (chan<- struct{}, <-chan nonceCheckResult) {
-	items := make([]pow.Block, len(headers))
-	for i, header := range headers {
-		items[i] = types.NewBlockWithHeader(header)
-	}
-	return verifyNonces(checker, items)
-}
-
-// verifyNoncesFromBlocks starts a concurrent block nonce verification,
-// returning a quit channel to abort the operations and a results channel
-// to retrieve the async verifications.
-func verifyNoncesFromBlocks(checker pow.PoW, blocks []*types.Block) (chan<- struct{}, <-chan nonceCheckResult) {
-	items := make([]pow.Block, len(blocks))
-	for i, block := range blocks {
-		items[i] = block
-	}
-	return verifyNonces(checker, items)
-}
-
-// verifyNonces starts a concurrent nonce verification, returning a quit channel
-// to abort the operations and a results channel to retrieve the async checks.
-func verifyNonces(checker pow.PoW, items []pow.Block) (chan<- struct{}, <-chan nonceCheckResult) {
-	// Spawn as many workers as allowed threads
-	workers := runtime.GOMAXPROCS(0)
-	if len(items) < workers {
-		workers = len(items)
-	}
-	// Create a task channel and spawn the verifiers
-	tasks := make(chan int, workers)
-	results := make(chan nonceCheckResult, len(items)) // Buffered to make sure all workers stop
-	for i := 0; i < workers; i++ {
-		go func() {
-			for index := range tasks {
-				results <- nonceCheckResult{index: index, valid: checker.Verify(items[index]) == nil}
-			}
-		}()
-	}
-	// Feed item indices to the workers until done or aborted
-	abort := make(chan struct{})
-	go func() {
-		defer close(tasks)
-
-		for i := range items {
-			select {
-			case tasks <- i:
-				continue
-			case <-abort:
-				return
-			}
-		}
-	}()
-	return abort, results
-}

core/chain_pow_test.go

@@ -1,238 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"errors"
-	"math/big"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/core/types"
-	"github.com/expanse-org/go-expanse/ethdb"
-	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
-)
-
-// failPow is a non-validating proof of work implementation, that returns true
-// from Verify for all but one block.
-type failPow struct {
-	failing uint64
-}
-
-func (pow failPow) Search(pow.Block, <-chan struct{}) (uint64, []byte) {
-	return 0, nil
-}
-func (pow failPow) Verify(block pow.Block) error {
-	if block.NumberU64() == pow.failing {
-		return errors.New("failed")
-	}
-	return nil
-}
-func (pow failPow) Hashrate() float64 { return 0 }
-
-// delayedPow is a non-validating proof of work implementation, that returns true
-// from Verify for all blocks, but delays them the configured amount of time.
-type delayedPow struct {
-	delay time.Duration
-}
-
-func (pow delayedPow) Search(pow.Block, <-chan struct{}) (uint64, []byte) {
-	return 0, nil
-}
-func (pow delayedPow) Verify(block pow.Block) error { time.Sleep(pow.delay); return nil }
-func (pow delayedPow) Hashrate() float64            { return 0 }
-
-// Tests that simple POW verification works, for both good and bad blocks.
-func TestPowVerification(t *testing.T) {
-	// Create a simple chain to verify
-	var (
-		testdb, _ = ethdb.NewMemDatabase()
-		genesis   = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
-		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 8, nil)
-	)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	// Run the POW checker for blocks one-by-one, checking for both valid and invalid nonces
-	for i := 0; i < len(blocks); i++ {
-		for j, full := range []bool{true, false} {
-			for k, valid := range []bool{true, false} {
-				var results <-chan nonceCheckResult
-
-				switch {
-				case full && valid:
-					_, results = verifyNoncesFromBlocks(pow.FakePow{}, []*types.Block{blocks[i]})
-				case full && !valid:
-					_, results = verifyNoncesFromBlocks(failPow{blocks[i].NumberU64()}, []*types.Block{blocks[i]})
-				case !full && valid:
-					_, results = verifyNoncesFromHeaders(pow.FakePow{}, []*types.Header{headers[i]})
-				case !full && !valid:
-					_, results = verifyNoncesFromHeaders(failPow{headers[i].Number.Uint64()}, []*types.Header{headers[i]})
-				}
-				// Wait for the verification result
-				select {
-				case result := <-results:
-					if result.index != 0 {
-						t.Errorf("test %d.%d.%d: invalid index: have %d, want 0", i, j, k, result.index)
-					}
-					if result.valid != valid {
-						t.Errorf("test %d.%d.%d: validity mismatch: have %v, want %v", i, j, k, result.valid, valid)
-					}
-				case <-time.After(time.Second):
-					t.Fatalf("test %d.%d.%d: verification timeout", i, j, k)
-				}
-				// Make sure no more data is returned
-				select {
-				case result := <-results:
-					t.Fatalf("test %d.%d.%d: unexpected result returned: %v", i, j, k, result)
-				case <-time.After(25 * time.Millisecond):
-				}
-			}
-		}
-	}
-}
-
-// Tests that concurrent POW verification works, for both good and bad blocks.
-func TestPowConcurrentVerification2(t *testing.T)  { testPowConcurrentVerification(t, 2) }
-func TestPowConcurrentVerification8(t *testing.T)  { testPowConcurrentVerification(t, 8) }
-func TestPowConcurrentVerification32(t *testing.T) { testPowConcurrentVerification(t, 32) }
-
-func testPowConcurrentVerification(t *testing.T, threads int) {
-	// Create a simple chain to verify
-	var (
-		testdb, _ = ethdb.NewMemDatabase()
-		genesis   = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
-		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 8, nil)
-	)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	// Set the number of threads to verify on
-	old := runtime.GOMAXPROCS(threads)
-	defer runtime.GOMAXPROCS(old)
-
-	// Run the POW checker for the entire block chain at once both for a valid and
-	// also an invalid chain (enough if one is invalid, last but one (arbitrary)).
-	for i, full := range []bool{true, false} {
-		for j, valid := range []bool{true, false} {
-			var results <-chan nonceCheckResult
-
-			switch {
-			case full && valid:
-				_, results = verifyNoncesFromBlocks(pow.FakePow{}, blocks)
-			case full && !valid:
-				_, results = verifyNoncesFromBlocks(failPow{uint64(len(blocks) - 1)}, blocks)
-			case !full && valid:
-				_, results = verifyNoncesFromHeaders(pow.FakePow{}, headers)
-			case !full && !valid:
-				_, results = verifyNoncesFromHeaders(failPow{uint64(len(headers) - 1)}, headers)
-			}
-			// Wait for all the verification results
-			checks := make(map[int]bool)
-			for k := 0; k < len(blocks); k++ {
-				select {
-				case result := <-results:
-					if _, ok := checks[result.index]; ok {
-						t.Fatalf("test %d.%d.%d: duplicate results for %d", i, j, k, result.index)
-					}
-					if result.index < 0 || result.index >= len(blocks) {
-						t.Fatalf("test %d.%d.%d: result %d out of bounds [%d, %d]", i, j, k, result.index, 0, len(blocks)-1)
-					}
-					checks[result.index] = result.valid
-
-				case <-time.After(time.Second):
-					t.Fatalf("test %d.%d.%d: verification timeout", i, j, k)
-				}
-			}
-			// Check nonce check validity
-			for k := 0; k < len(blocks); k++ {
-				want := valid || (k != len(blocks)-2) // We chose the last but one nonce in the chain to fail
-				if checks[k] != want {
-					t.Errorf("test %d.%d.%d: validity mismatch: have %v, want %v", i, j, k, checks[k], want)
-				}
-			}
-			// Make sure no more data is returned
-			select {
-			case result := <-results:
-				t.Fatalf("test %d.%d: unexpected result returned: %v", i, j, result)
-			case <-time.After(25 * time.Millisecond):
-			}
-		}
-	}
-}
-
-// Tests that aborting a POW validation indeed prevents further checks from being
-// run, as well as checks that no left-over goroutines are leaked.
-func TestPowConcurrentAbortion2(t *testing.T)  { testPowConcurrentAbortion(t, 2) }
-func TestPowConcurrentAbortion8(t *testing.T)  { testPowConcurrentAbortion(t, 8) }
-func TestPowConcurrentAbortion32(t *testing.T) { testPowConcurrentAbortion(t, 32) }
-
-func testPowConcurrentAbortion(t *testing.T, threads int) {
-	// Create a simple chain to verify
-	var (
-		testdb, _ = ethdb.NewMemDatabase()
-		genesis   = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
-		blocks, _ = GenerateChain(params.TestChainConfig, genesis, testdb, 1024, nil)
-	)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	// Set the number of threads to verify on
-	old := runtime.GOMAXPROCS(threads)
-	defer runtime.GOMAXPROCS(old)
-
-	// Run the POW checker for the entire block chain at once
-	for i, full := range []bool{true, false} {
-		var abort chan<- struct{}
-		var results <-chan nonceCheckResult
-
-		// Start the verifications and immediately abort
-		if full {
-			abort, results = verifyNoncesFromBlocks(delayedPow{time.Millisecond}, blocks)
-		} else {
-			abort, results = verifyNoncesFromHeaders(delayedPow{time.Millisecond}, headers)
-		}
-		close(abort)
-
-		// Deplete the results channel
-		verified := make(map[int]struct{})
-		for depleted := false; !depleted; {
-			select {
-			case result := <-results:
-				verified[result.index] = struct{}{}
-			case <-time.After(50 * time.Millisecond):
-				depleted = true
-			}
-		}
-		// Check that abortion was honored by not processing too many POWs
-		if len(verified) > 2*threads {
-			t.Errorf("test %d: verification count too large: have %d, want below %d", i, len(verified), 2*threads)
-		}
-		// Check that there are no gaps in the results
-		for j := 0; j < len(verified); j++ {
-			if _, ok := verified[j]; !ok {
-				t.Errorf("test %d.%d: gap found in verification results", i, j)
-			}
-		}
-	}
-}

core/dao.go

@@ -18,6 +18,7 @@ package core
 
 import (
 	"bytes"
+	"fmt"
 	"math/big"
 
 	"github.com/expanse-org/go-expanse/core/state"
@@ -46,11 +47,11 @@ func ValidateDAOHeaderExtraData(config *params.ChainConfig, header *types.Header
 	// Depending whether we support or oppose the fork, validate the extra-data contents
 	if config.DAOForkSupport {
 		if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
-			return ValidationError("DAO pro-fork bad block extra-data: 0x%x", header.Extra)
+			return fmt.Errorf("DAO pro-fork bad block extra-data: 0x%x", header.Extra)
 		}
 	} else {
 		if bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
-			return ValidationError("DAO no-fork bad block extra-data: 0x%x", header.Extra)
+			return fmt.Errorf("DAO no-fork bad block extra-data: 0x%x", header.Extra)
 		}
 	}
 	// All ok, header has the same extra-data we expect

core/dao_test.go

@@ -20,11 +20,11 @@ import (
 	"math/big"
 	"testing"
 
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 // Tests that DAO-fork enabled clients can properly filter out fork-commencing
@@ -42,12 +42,12 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	proDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(proDb)
 	proConf := &params.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true}
-	proBc, _ := NewBlockChain(proDb, proConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	proBc, _ := NewBlockChain(proDb, proConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
 	conDb, _ := ethdb.NewMemDatabase()
 	gspec.MustCommit(conDb)
 	conConf := &params.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false}
-	conBc, _ := NewBlockChain(conDb, conConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	conBc, _ := NewBlockChain(conDb, conConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
 	if _, err := proBc.InsertChain(prefix); err != nil {
 		t.Fatalf("pro-fork: failed to import chain prefix: %v", err)
@@ -60,9 +60,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a pro-fork block, and try to feed into the no-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ := NewBlockChain(db, conConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+		bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
-		blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1))
+		blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
 		for j := 0; j < len(blocks)/2; j++ {
 			blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j]
 		}
@@ -81,9 +81,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a no-fork block, and try to feed into the pro-fork chain
 		db, _ = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ = NewBlockChain(db, proConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+		bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
-		blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1))
+		blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
 		for j := 0; j < len(blocks)/2; j++ {
 			blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j]
 		}
@@ -103,9 +103,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ := NewBlockChain(db, conConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
-	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()+1))
+	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
 	for j := 0; j < len(blocks)/2; j++ {
 		blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j]
 	}
@@ -119,9 +119,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
 	db, _ = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ = NewBlockChain(db, proConf, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 
-	blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()+1))
+	blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
 	for j := 0; j < len(blocks)/2; j++ {
 		blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j]
 	}

core/database_util_test.go

@@ -18,14 +18,12 @@ package core
 
 import (
 	"bytes"
-	"encoding/json"
 	"io/ioutil"
 	"math/big"
 	"os"
 	"testing"
 
 	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/common/math"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/crypto"
 	"github.com/expanse-org/go-expanse/crypto/sha3"
@@ -34,58 +32,6 @@ import (
 	"github.com/expanse-org/go-expanse/rlp"
 )
 
-type diffTest struct {
-	ParentTimestamp    uint64
-	ParentDifficulty   *big.Int
-	CurrentTimestamp   uint64
-	CurrentBlocknumber *big.Int
-	CurrentDifficulty  *big.Int
-}
-
-func (d *diffTest) UnmarshalJSON(b []byte) (err error) {
-	var ext struct {
-		ParentTimestamp    string
-		ParentDifficulty   string
-		CurrentTimestamp   string
-		CurrentBlocknumber string
-		CurrentDifficulty  string
-	}
-	if err := json.Unmarshal(b, &ext); err != nil {
-		return err
-	}
-
-	d.ParentTimestamp = math.MustParseUint64(ext.ParentTimestamp)
-	d.ParentDifficulty = math.MustParseBig256(ext.ParentDifficulty)
-	d.CurrentTimestamp = math.MustParseUint64(ext.CurrentTimestamp)
-	d.CurrentBlocknumber = math.MustParseBig256(ext.CurrentBlocknumber)
-	d.CurrentDifficulty = math.MustParseBig256(ext.CurrentDifficulty)
-
-	return nil
-}
-
-func TestCalcDifficulty(t *testing.T) {
-	file, err := os.Open("../tests/files/BasicTests/difficulty.json")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer file.Close()
-
-	tests := make(map[string]diffTest)
-	err = json.NewDecoder(file).Decode(&tests)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	config := &params.ChainConfig{HomesteadBlock: big.NewInt(1150000)}
-	for name, test := range tests {
-		number := new(big.Int).Sub(test.CurrentBlocknumber, big.NewInt(1))
-		diff := CalcDifficulty(config, test.CurrentTimestamp, test.ParentTimestamp, number, test.ParentDifficulty)
-		if diff.Cmp(test.CurrentDifficulty) != 0 {
-			t.Error(name, "failed. Expected", test.CurrentDifficulty, "and calculated", diff)
-		}
-	}
-}
-
 // Tests block header storage and retrieval operations.
 func TestHeaderStorage(t *testing.T) {
 	db, _ := ethdb.NewMemDatabase()
@@ -562,7 +508,11 @@ func TestMipmapChain(t *testing.T) {
 	)
 	defer db.Close()
 
-	genesis := testGenesis(addr, big.NewInt(1000000)).MustCommit(db)
+	gspec := &Genesis{
+		Config: params.TestChainConfig,
+		Alloc:  GenesisAlloc{addr: {Balance: big.NewInt(1000000)}},
+	}
+	genesis := gspec.MustCommit(db)
 	chain, receipts := GenerateChain(params.TestChainConfig, genesis, db, 1010, func(i int, gen *BlockGen) {
 		var receipts types.Receipts
 		switch i {

core/error.go

@@ -16,188 +16,16 @@
 
 package core
 
-import (
-	"errors"
-	"fmt"
-	"math/big"
-
-	"github.com/expanse-org/go-expanse/common"
-)
+import "errors"
 
 var (
-	BlockNumberErr   = errors.New("block number invalid")
-	BlockFutureErr   = errors.New("block time is in the future")
-	BlockTSTooBigErr = errors.New("block time too big")
-	BlockEqualTSErr  = errors.New("block time stamp equal to previous")
+	// ErrKnownBlock is returned when a block to import is already known locally.
+	ErrKnownBlock = errors.New("block already known")
+
+	// ErrGasLimitReached is returned by the gas pool if the amount of gas required
+	// by a transaction is higher than what's left in the block.
+	ErrGasLimitReached = errors.New("gas limit reached")
+
+	// ErrBlacklistedHash is returned if a block to import is on the blacklist.
+	ErrBlacklistedHash = errors.New("blacklisted hash")
 )
-
-// Parent error. In case a parent is unknown this error will be thrown
-// by the block manager
-type ParentErr struct {
-	Message string
-}
-
-func (err *ParentErr) Error() string {
-	return err.Message
-}
-
-func ParentError(hash common.Hash) error {
-	return &ParentErr{Message: fmt.Sprintf("Block's parent unknown %x", hash)}
-}
-
-func IsParentErr(err error) bool {
-	_, ok := err.(*ParentErr)
-	return ok
-}
-
-type UncleErr struct {
-	Message string
-}
-
-func (err *UncleErr) Error() string {
-	return err.Message
-}
-
-func UncleError(format string, v ...interface{}) error {
-	return &UncleErr{Message: fmt.Sprintf(format, v...)}
-}
-
-func IsUncleErr(err error) bool {
-	_, ok := err.(*UncleErr)
-	return ok
-}
-
-// Block validation error. If any validation fails, this error will be thrown
-type ValidationErr struct {
-	Message string
-}
-
-func (err *ValidationErr) Error() string {
-	return err.Message
-}
-
-func ValidationError(format string, v ...interface{}) *ValidationErr {
-	return &ValidationErr{Message: fmt.Sprintf(format, v...)}
-}
-
-func IsValidationErr(err error) bool {
-	_, ok := err.(*ValidationErr)
-	return ok
-}
-
-type NonceErr struct {
-	Message string
-	Is, Exp uint64
-}
-
-func (err *NonceErr) Error() string {
-	return err.Message
-}
-
-func NonceError(is, exp uint64) *NonceErr {
-	return &NonceErr{Message: fmt.Sprintf("Transaction w/ invalid nonce. tx=%d  state=%d)", is, exp), Is: is, Exp: exp}
-}
-
-func IsNonceErr(err error) bool {
-	_, ok := err.(*NonceErr)
-	return ok
-}
-
-// BlockNonceErr indicates that a block's nonce is invalid.
-type BlockNonceErr struct {
-	Number *big.Int
-	Hash   common.Hash
-	Nonce  uint64
-}
-
-func (err *BlockNonceErr) Error() string {
-	return fmt.Sprintf("nonce for #%d [%x…] is invalid (got %d)", err.Number, err.Hash, err.Nonce)
-}
-
-// IsBlockNonceErr returns true for invalid block nonce errors.
-func IsBlockNonceErr(err error) bool {
-	_, ok := err.(*BlockNonceErr)
-	return ok
-}
-
-type InvalidTxErr struct {
-	Message string
-}
-
-func (err *InvalidTxErr) Error() string {
-	return err.Message
-}
-
-func InvalidTxError(err error) *InvalidTxErr {
-	return &InvalidTxErr{fmt.Sprintf("%v", err)}
-}
-
-func IsInvalidTxErr(err error) bool {
-	_, ok := err.(*InvalidTxErr)
-	return ok
-}
-
-type TDError struct {
-	a, b *big.Int
-}
-
-func (self *TDError) Error() string {
-	return fmt.Sprintf("incoming chain has a lower or equal TD (%v <= %v)", self.a, self.b)
-}
-func IsTDError(e error) bool {
-	_, ok := e.(*TDError)
-	return ok
-}
-
-type KnownBlockError struct {
-	number *big.Int
-	hash   common.Hash
-}
-
-func (self *KnownBlockError) Error() string {
-	return fmt.Sprintf("block %v already known (%x)", self.number, self.hash[0:4])
-}
-func IsKnownBlockErr(e error) bool {
-	_, ok := e.(*KnownBlockError)
-	return ok
-}
-
-type ValueTransferError struct {
-	message string
-}
-
-func ValueTransferErr(str string, v ...interface{}) *ValueTransferError {
-	return &ValueTransferError{fmt.Sprintf(str, v...)}
-}
-
-func (self *ValueTransferError) Error() string {
-	return self.message
-}
-func IsValueTransferErr(e error) bool {
-	_, ok := e.(*ValueTransferError)
-	return ok
-}
-
-type BadHashError common.Hash
-
-func (h BadHashError) Error() string {
-	return fmt.Sprintf("Found known bad hash in chain %x", h[:])
-}
-
-func IsBadHashError(err error) bool {
-	_, ok := err.(BadHashError)
-	return ok
-}
-
-type GasLimitErr struct {
-	Have, Want *big.Int
-}
-
-func IsGasLimitErr(err error) bool {
-	_, ok := err.(*GasLimitErr)
-	return ok
-}
-
-func (err *GasLimitErr) Error() string {
-	return fmt.Sprintf("GasLimit reached. Have %d gas, transaction requires %d", err.Have, err.Want)
-}

core/evm.go

@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2016 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -20,25 +20,36 @@ import (
 	"math/big"
 
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/core/vm"
 )
 
-// BlockFetcher retrieves headers by their hash
-type HeaderFetcher interface {
-	// GetHeader returns the hash corresponding to their hash
+// ChainContext supports retrieving headers and consensus parameters from the
+// current blockchain to be used during transaction processing.
+type ChainContext interface {
+	// Engine retrieves the chain's consensus engine.
+	Engine() consensus.Engine
+
+	// GetHeader returns the hash corresponding to their hash.
 	GetHeader(common.Hash, uint64) *types.Header
 }
 
 // NewEVMContext creates a new context for use in the EVM.
-func NewEVMContext(msg Message, header *types.Header, chain HeaderFetcher) vm.Context {
+func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author *common.Address) vm.Context {
+	// If we don't have an explicit author (i.e. not mining), extract from the header
+	var beneficiary common.Address
+	if author == nil {
+		beneficiary, _ = chain.Engine().Author(header) // Ignore error, we're past header validation
+	} else {
+		beneficiary = *author
+	}
 	return vm.Context{
 		CanTransfer: CanTransfer,
 		Transfer:    Transfer,
 		GetHash:     GetHashFn(header, chain),
-
 		Origin:      msg.From(),
-		Coinbase:    header.Coinbase,
+		Coinbase:    beneficiary,
 		BlockNumber: new(big.Int).Set(header.Number),
 		Time:        new(big.Int).Set(header.Time),
 		Difficulty:  new(big.Int).Set(header.Difficulty),
@@ -48,7 +59,7 @@ func NewEVMContext(msg Message, header *types.Header, chain HeaderFetcher) vm.Co
 }
 
 // GetHashFn returns a GetHashFunc which retrieves header hashes by number
-func GetHashFn(ref *types.Header, chain HeaderFetcher) func(n uint64) common.Hash {
+func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash {
 	return func(n uint64) common.Hash {
 		for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) {
 			if header.Number.Uint64() == n {

core/gaspool.go

@@ -35,7 +35,7 @@ func (gp *GasPool) AddGas(amount *big.Int) *GasPool {
 func (gp *GasPool) SubGas(amount *big.Int) error {
 	i := (*big.Int)(gp)
 	if i.Cmp(amount) < 0 {
-		return &GasLimitErr{Have: new(big.Int).Set(i), Want: amount}
+		return ErrGasLimitReached
 	}
 	i.Sub(i, amount)
 	return nil

core/gen_genesis.go

@@ -14,19 +14,19 @@ import (
 )
 
 func (g Genesis) MarshalJSON() ([]byte, error) {
-	type GenesisJSON struct {
-		Config     *params.ChainConfig                         `json:"config" optional:"true"`
-		Nonce      math.HexOrDecimal64                         `json:"nonce" optional:"true"`
-		Timestamp  math.HexOrDecimal64                         `json:"timestamp" optional:"true"`
-		ParentHash common.Hash                                 `json:"parentHash" optional:"true"`
-		ExtraData  hexutil.Bytes                               `json:"extraData" optional:"true"`
-		GasLimit   math.HexOrDecimal64                         `json:"gasLimit"`
-		Difficulty *math.HexOrDecimal256                       `json:"difficulty"`
-		Mixhash    common.Hash                                 `json:"mixHash" optional:"true"`
-		Coinbase   common.Address                              `json:"coinbase" optional:"true"`
-		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"`
+	type Genesis struct {
+		Config     *params.ChainConfig                         `json:"config"`
+		Nonce      math.HexOrDecimal64                         `json:"nonce"`
+		Timestamp  math.HexOrDecimal64                         `json:"timestamp"`
+		ParentHash common.Hash                                 `json:"parentHash"`
+		ExtraData  hexutil.Bytes                               `json:"extraData"`
+		GasLimit   math.HexOrDecimal64                         `json:"gasLimit"   gencodec:"required"`
+		Difficulty *math.HexOrDecimal256                       `json:"difficulty" gencodec:"required"`
+		Mixhash    common.Hash                                 `json:"mixHash"`
+		Coinbase   common.Address                              `json:"coinbase"`
+		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"      gencodec:"required"`
 	}
-	var enc GenesisJSON
+	var enc Genesis
 	enc.Config = g.Config
 	enc.Nonce = math.HexOrDecimal64(g.Nonce)
 	enc.Timestamp = math.HexOrDecimal64(g.Timestamp)
@@ -46,59 +46,57 @@ func (g Genesis) MarshalJSON() ([]byte, error) {
 }
 
 func (g *Genesis) UnmarshalJSON(input []byte) error {
-	type GenesisJSON struct {
-		Config     *params.ChainConfig                         `json:"config" optional:"true"`
-		Nonce      *math.HexOrDecimal64                        `json:"nonce" optional:"true"`
-		Timestamp  *math.HexOrDecimal64                        `json:"timestamp" optional:"true"`
-		ParentHash *common.Hash                                `json:"parentHash" optional:"true"`
-		ExtraData  hexutil.Bytes                               `json:"extraData" optional:"true"`
-		GasLimit   *math.HexOrDecimal64                        `json:"gasLimit"`
-		Difficulty *math.HexOrDecimal256                       `json:"difficulty"`
-		Mixhash    *common.Hash                                `json:"mixHash" optional:"true"`
-		Coinbase   *common.Address                             `json:"coinbase" optional:"true"`
-		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"`
+	type Genesis struct {
+		Config     *params.ChainConfig                         `json:"config"`
+		Nonce      *math.HexOrDecimal64                        `json:"nonce"`
+		Timestamp  *math.HexOrDecimal64                        `json:"timestamp"`
+		ParentHash *common.Hash                                `json:"parentHash"`
+		ExtraData  hexutil.Bytes                               `json:"extraData"`
+		GasLimit   *math.HexOrDecimal64                        `json:"gasLimit"   gencodec:"required"`
+		Difficulty *math.HexOrDecimal256                       `json:"difficulty" gencodec:"required"`
+		Mixhash    *common.Hash                                `json:"mixHash"`
+		Coinbase   *common.Address                             `json:"coinbase"`
+		Alloc      map[common.UnprefixedAddress]GenesisAccount `json:"alloc"      gencodec:"required"`
 	}
-	var dec GenesisJSON
+	var dec Genesis
 	if err := json.Unmarshal(input, &dec); err != nil {
 		return err
 	}
-	var x Genesis
 	if dec.Config != nil {
-		x.Config = dec.Config
+		g.Config = dec.Config
 	}
 	if dec.Nonce != nil {
-		x.Nonce = uint64(*dec.Nonce)
+		g.Nonce = uint64(*dec.Nonce)
 	}
 	if dec.Timestamp != nil {
-		x.Timestamp = uint64(*dec.Timestamp)
+		g.Timestamp = uint64(*dec.Timestamp)
 	}
 	if dec.ParentHash != nil {
-		x.ParentHash = *dec.ParentHash
+		g.ParentHash = *dec.ParentHash
 	}
 	if dec.ExtraData != nil {
-		x.ExtraData = dec.ExtraData
+		g.ExtraData = dec.ExtraData
 	}
 	if dec.GasLimit == nil {
 		return errors.New("missing required field 'gasLimit' for Genesis")
 	}
-	x.GasLimit = uint64(*dec.GasLimit)
+	g.GasLimit = uint64(*dec.GasLimit)
 	if dec.Difficulty == nil {
 		return errors.New("missing required field 'difficulty' for Genesis")
 	}
-	x.Difficulty = (*big.Int)(dec.Difficulty)
+	g.Difficulty = (*big.Int)(dec.Difficulty)
 	if dec.Mixhash != nil {
-		x.Mixhash = *dec.Mixhash
+		g.Mixhash = *dec.Mixhash
 	}
 	if dec.Coinbase != nil {
-		x.Coinbase = *dec.Coinbase
+		g.Coinbase = *dec.Coinbase
 	}
 	if dec.Alloc == nil {
 		return errors.New("missing required field 'alloc' for Genesis")
 	}
-	x.Alloc = make(GenesisAlloc, len(dec.Alloc))
+	g.Alloc = make(GenesisAlloc, len(dec.Alloc))
 	for k, v := range dec.Alloc {
-		x.Alloc[common.Address(k)] = v
+		g.Alloc[common.Address(k)] = v
 	}
-	*g = x
 	return nil
 }

core/gen_genesis_account.go

@@ -13,13 +13,13 @@ import (
 )
 
 func (g GenesisAccount) MarshalJSON() ([]byte, error) {
-	type GenesisAccountJSON struct {
-		Code    hexutil.Bytes               `json:"code,omitempty" optional:"true"`
-		Storage map[common.Hash]common.Hash `json:"storage,omitempty" optional:"true"`
-		Balance *math.HexOrDecimal256       `json:"balance"`
-		Nonce   math.HexOrDecimal64         `json:"nonce,omitempty" optional:"true"`
+	type GenesisAccount struct {
+		Code    hexutil.Bytes               `json:"code,omitempty"`
+		Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
+		Balance *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
+		Nonce   math.HexOrDecimal64         `json:"nonce,omitempty"`
 	}
-	var enc GenesisAccountJSON
+	var enc GenesisAccount
 	enc.Code = g.Code
 	enc.Storage = g.Storage
 	enc.Balance = (*math.HexOrDecimal256)(g.Balance)
@@ -28,30 +28,28 @@ func (g GenesisAccount) MarshalJSON() ([]byte, error) {
 }
 
 func (g *GenesisAccount) UnmarshalJSON(input []byte) error {
-	type GenesisAccountJSON struct {
-		Code    hexutil.Bytes               `json:"code,omitempty" optional:"true"`
-		Storage map[common.Hash]common.Hash `json:"storage,omitempty" optional:"true"`
-		Balance *math.HexOrDecimal256       `json:"balance"`
-		Nonce   *math.HexOrDecimal64        `json:"nonce,omitempty" optional:"true"`
+	type GenesisAccount struct {
+		Code    hexutil.Bytes               `json:"code,omitempty"`
+		Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
+		Balance *math.HexOrDecimal256       `json:"balance" gencodec:"required"`
+		Nonce   *math.HexOrDecimal64        `json:"nonce,omitempty"`
 	}
-	var dec GenesisAccountJSON
+	var dec GenesisAccount
 	if err := json.Unmarshal(input, &dec); err != nil {
 		return err
 	}
-	var x GenesisAccount
 	if dec.Code != nil {
-		x.Code = dec.Code
+		g.Code = dec.Code
 	}
 	if dec.Storage != nil {
-		x.Storage = dec.Storage
+		g.Storage = dec.Storage
 	}
 	if dec.Balance == nil {
 		return errors.New("missing required field 'balance' for GenesisAccount")
 	}
-	x.Balance = (*big.Int)(dec.Balance)
+	g.Balance = (*big.Int)(dec.Balance)
 	if dec.Nonce != nil {
-		x.Nonce = uint64(*dec.Nonce)
+		g.Nonce = uint64(*dec.Nonce)
 	}
-	*g = x
 	return nil
 }

core/genesis.go

@@ -41,16 +41,16 @@ var errGenesisNoConfig = errors.New("genesis has no chain configuration")
 // Genesis specifies the header fields, state of a genesis block. It also defines hard
 // fork switch-over blocks through the chain configuration.
 type Genesis struct {
-	Config     *params.ChainConfig `json:"config" optional:"true"`
-	Nonce      uint64              `json:"nonce" optional:"true"`
-	Timestamp  uint64              `json:"timestamp" optional:"true"`
-	ParentHash common.Hash         `json:"parentHash" optional:"true"`
-	ExtraData  []byte              `json:"extraData" optional:"true"`
-	GasLimit   uint64              `json:"gasLimit"`
-	Difficulty *big.Int            `json:"difficulty"`
-	Mixhash    common.Hash         `json:"mixHash" optional:"true"`
-	Coinbase   common.Address      `json:"coinbase" optional:"true"`
-	Alloc      GenesisAlloc        `json:"alloc"`
+	Config     *params.ChainConfig `json:"config"`
+	Nonce      uint64              `json:"nonce"`
+	Timestamp  uint64              `json:"timestamp"`
+	ParentHash common.Hash         `json:"parentHash"`
+	ExtraData  []byte              `json:"extraData"`
+	GasLimit   uint64              `json:"gasLimit"   gencodec:"required"`
+	Difficulty *big.Int            `json:"difficulty" gencodec:"required"`
+	Mixhash    common.Hash         `json:"mixHash"`
+	Coinbase   common.Address      `json:"coinbase"`
+	Alloc      GenesisAlloc        `json:"alloc"      gencodec:"required"`
 }
 
 // GenesisAlloc specifies the initial state that is part of the genesis block.
@@ -58,10 +58,10 @@ type GenesisAlloc map[common.Address]GenesisAccount
 
 // GenesisAccount is an account in the state of the genesis block.
 type GenesisAccount struct {
-	Code    []byte                      `json:"code,omitempty" optional:"true"`
-	Storage map[common.Hash]common.Hash `json:"storage,omitempty" optional:"true"`
-	Balance *big.Int                    `json:"balance"`
-	Nonce   uint64                      `json:"nonce,omitempty" optional:"true"`
+	Code    []byte                      `json:"code,omitempty"`
+	Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
+	Balance *big.Int                    `json:"balance" gencodec:"required"`
+	Nonce   uint64                      `json:"nonce,omitempty"`
 }
 
 // field type overrides for gencodec

core/genesis_test.go

@@ -22,12 +22,13 @@ import (
 	"testing"
 
 	"github.com/davecgh/go-spew/spew"
+
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core/vm"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 func TestDefaultGenesisBlock(t *testing.T) {
@@ -119,7 +120,7 @@ func TestSetupGenesis(t *testing.T) {
 				// Commit the 'old' genesis block with Homestead transition at #2.
 				// Advance to block #4, past the homestead transition block of customg.
 				genesis := oldcustomg.MustCommit(db)
-				bc, _ := NewBlockChain(db, oldcustomg.Config, pow.FakePow{}, new(event.TypeMux), vm.Config{})
+				bc, _ := NewBlockChain(db, oldcustomg.Config, ethash.NewFullFaker(), new(event.TypeMux), vm.Config{})
 				bc.SetValidator(bproc{})
 				bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0))
 				bc.CurrentBlock()

core/headerchain.go

@@ -18,21 +18,19 @@ package core
 
 import (
 	crand "crypto/rand"
+	"errors"
 	"fmt"
 	"math"
 	"math/big"
 	mrand "math/rand"
-	"runtime"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/core/types"
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 	"github.com/hashicorp/golang-lru"
 )
 
@@ -62,18 +60,15 @@ type HeaderChain struct {
 
 	procInterrupt func() bool
 
-	rand         *mrand.Rand
-	getValidator getHeaderValidatorFn
+	rand   *mrand.Rand
+	engine consensus.Engine
 }
 
-// getHeaderValidatorFn returns a HeaderValidator interface
-type getHeaderValidatorFn func() HeaderValidator
-
 // NewHeaderChain creates a new HeaderChain structure.
 //  getValidator should return the parent's validator
 //  procInterrupt points to the parent's interrupt semaphore
 //  wg points to the parent's shutdown wait group
-func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, getValidator getHeaderValidatorFn, procInterrupt func() bool) (*HeaderChain, error) {
+func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, procInterrupt func() bool) (*HeaderChain, error) {
 	headerCache, _ := lru.New(headerCacheLimit)
 	tdCache, _ := lru.New(tdCacheLimit)
 	numberCache, _ := lru.New(numberCacheLimit)
@@ -92,7 +87,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, getValid
 		numberCache:   numberCache,
 		procInterrupt: procInterrupt,
 		rand:          mrand.New(mrand.NewSource(seed.Int64())),
-		getValidator:  getValidator,
+		engine:        engine,
 	}
 
 	hc.genesisHeader = hc.GetHeaderByNumber(0)
@@ -142,7 +137,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 	// Calculate the total difficulty of the header
 	ptd := hc.GetTd(header.ParentHash, number-1)
 	if ptd == nil {
-		return NonStatTy, ParentError(header.ParentHash)
+		return NonStatTy, consensus.ErrUnknownAncestor
 	}
 	localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64())
 	externTd := new(big.Int).Add(header.Difficulty, ptd)
@@ -228,78 +223,34 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int)
 		}
 	}
 
-	// Generate the list of headers that should be POW verified
-	verify := make([]bool, len(chain))
-	for i := 0; i < len(verify)/checkFreq; i++ {
+	// Generate the list of seal verification requests, and start the parallel verifier
+	seals := make([]bool, len(chain))
+	for i := 0; i < len(seals)/checkFreq; i++ {
 		index := i*checkFreq + hc.rand.Intn(checkFreq)
-		if index >= len(verify) {
-			index = len(verify) - 1
+		if index >= len(seals) {
+			index = len(seals) - 1
 		}
-		verify[index] = true
+		seals[index] = true
 	}
-	verify[len(verify)-1] = true // Last should always be verified to avoid junk
+	seals[len(seals)-1] = true // Last should always be verified to avoid junk
 
-	// Create the header verification task queue and worker functions
-	tasks := make(chan int, len(chain))
-	for i := 0; i < len(chain); i++ {
-		tasks <- i
-	}
-	close(tasks)
+	abort, results := hc.engine.VerifyHeaders(hc, chain, seals)
+	defer close(abort)
 
-	errs, failed := make([]error, len(tasks)), int32(0)
-	process := func(worker int) {
-		for index := range tasks {
-			header, hash := chain[index], chain[index].Hash()
-
-			// Short circuit insertion if shutting down or processing failed
-			if hc.procInterrupt() {
-				return
-			}
-			if atomic.LoadInt32(&failed) > 0 {
-				return
-			}
-			// Short circuit if the header is bad or already known
-			if BadHashes[hash] {
-				errs[index] = BadHashError(hash)
-				atomic.AddInt32(&failed, 1)
-				return
-			}
-			if hc.HasHeader(hash) {
-				continue
-			}
-			// Verify that the header honors the chain parameters
-			checkPow := verify[index]
-
-			var err error
-			if index == 0 {
-				err = hc.getValidator().ValidateHeader(header, hc.GetHeader(header.ParentHash, header.Number.Uint64()-1), checkPow)
-			} else {
-				err = hc.getValidator().ValidateHeader(header, chain[index-1], checkPow)
-			}
-			if err != nil {
-				errs[index] = err
-				atomic.AddInt32(&failed, 1)
-				return
-			}
+	// Iterate over the headers and ensure they all check out
+	for i, header := range chain {
+		// If the chain is terminating, stop processing blocks
+		if hc.procInterrupt() {
+			log.Debug("Premature abort during headers verification")
+			return 0, errors.New("aborted")
 		}
-	}
-	// Start as many worker threads as goroutines allowed
-	pending := new(sync.WaitGroup)
-	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
-		pending.Add(1)
-		go func(id int) {
-			defer pending.Done()
-			process(id)
-		}(i)
-	}
-	pending.Wait()
-
-	// If anything failed, report
-	if failed > 0 {
-		for i, err := range errs {
-			if err != nil {
-				return i, err
-			}
+		// If the header is a banned one, straight out abort
+		if BadHashes[header.Hash()] {
+			return i, ErrBlacklistedHash
+		}
+		// Otherwise wait for headers checks and ensure they pass
+		if err := <-results; err != nil {
+			return i, err
 		}
 	}
 
@@ -313,13 +264,11 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa
 	for i, header := range chain {
 		// Short circuit insertion if shutting down
 		if hc.procInterrupt() {
-			log.Debug("Premature abort during headers processing")
-			break
+			log.Debug("Premature abort during headers import")
+			return i, errors.New("aborted")
 		}
-		hash := header.Hash()
-
 		// If the header's already known, skip it, otherwise store
-		if hc.HasHeader(hash) {
+		if hc.GetHeader(header.Hash(), header.Number.Uint64()) != nil {
 			stats.ignored++
 			continue
 		}
@@ -490,35 +439,14 @@ func (hc *HeaderChain) SetGenesis(head *types.Header) {
 	hc.genesisHeader = head
 }
 
-// headerValidator is responsible for validating block headers
-//
-// headerValidator implements HeaderValidator.
-type headerValidator struct {
-	config *params.ChainConfig
-	hc     *HeaderChain // Canonical header chain
-	Pow    pow.PoW      // Proof of work used for validating
-}
+// Config retrieves the header chain's chain configuration.
+func (hc *HeaderChain) Config() *params.ChainConfig { return hc.config }
 
-// NewBlockValidator returns a new block validator which is safe for re-use
-func NewHeaderValidator(config *params.ChainConfig, chain *HeaderChain, pow pow.PoW) HeaderValidator {
-	return &headerValidator{
-		config: config,
-		Pow:    pow,
-		hc:     chain,
-	}
-}
+// Engine retrieves the header chain's consensus engine.
+func (hc *HeaderChain) Engine() consensus.Engine { return hc.engine }
 
-// ValidateHeader validates the given header and, depending on the pow arg,
-// checks the proof of work of the given header. Returns an error if the
-// validation failed.
-func (v *headerValidator) ValidateHeader(header, parent *types.Header, checkPow bool) error {
-	// Short circuit if the parent is missing.
-	if parent == nil {
-		return ParentError(header.ParentHash)
-	}
-	// Short circuit if the header's already known or its parent missing
-	if v.hc.HasHeader(header.Hash()) {
-		return nil
-	}
-	return ValidateHeader(v.config, v.Pow, header, parent, checkPow, false)
+// GetBlock implements consensus.ChainReader, and returns nil for every input as
+// a header chain does not have blocks available for retrieval.
+func (hc *HeaderChain) GetBlock(hash common.Hash, number uint64) *types.Block {
+	return nil
 }

core/state/journal.go

@@ -73,6 +73,7 @@ type (
 	touchChange struct {
 		account *common.Address
 		prev    bool
+		prevDirty bool
 	}
 )
 
@@ -97,8 +98,10 @@ var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
 
 func (ch touchChange) undo(s *StateDB) {
 	if !ch.prev && *ch.account != ripemd {
-		delete(s.stateObjects, *ch.account)
-		delete(s.stateObjectsDirty, *ch.account)
+		s.getStateObject(*ch.account).touched = ch.prev
+		if !ch.prevDirty {
+			delete(s.stateObjectsDirty, *ch.account)
+		}
 	}
 }