Merge pull request #19 from expanse-org/v1.6.x

V1.6.6

.mailmap

@@ -73,7 +73,7 @@ RJ Catalano <rj@erisindustries.com>
 
 Nchinda Nchinda <nchinda2@gmail.com>
 
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru> <homotopycolimit@users.noreply.github.com>
 
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 
@@ -94,3 +94,22 @@ Nick Johnson <arachnid@notdot.net>
 Henning Diedrich <hd@eonblast.com>
 Henning Diedrich <hd@eonblast.com> Drake Burroughs <wildfyre@hotmail.com>
 
+Felix Lange <fjl@twurst.com>
+Felix Lange <fjl@twurst.com> <fjl@users.noreply.github.com>
+
+Максим Чусовлянов <mchusovlianov@gmail.com>
+
+Louis Holbrook <dev@holbrook.no>
+Louis Holbrook <dev@holbrook.no> <nolash@users.noreply.github.com>
+
+Thomas Bocek <tom@tomp2p.net>
+
+Victor Tran <vu.tran54@gmail.com>
+
+Justin Drake <drakefjustin@gmail.com>
+
+Frank Wang <eternnoir@gmail.com>
+
+Gary Rong <garyrong0905@gmail.com>
+
+Guillaume Nicolas <guin56@gmail.com>

.travis.yml

@@ -3,11 +3,10 @@ go_import_path: github.com/expanse-org/go-expanse
 sudo: false
 matrix:
   include:
-
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.8
+      go: 1.8.3
       script:
         - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
@@ -16,14 +15,13 @@ matrix:
         - go run build/ci.go install
 
     - os: osx
-      go: 1.8
+      go: 1.8.3
       sudo: required
       script:
         - brew update
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-
 install:
   - go get golang.org/x/tools/cmd/cover
 script:

AUTHORS

@@ -3,24 +3,30 @@
 Ales Katona <ales@coinbase.com>
 Alex Leverington <alex@ethdev.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+Brian Schroeder <bts@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
 Daniel A. Nagy <nagy.da@gmail.com>
+Diego Siqueira <DiSiqueira@users.noreply.github.com>
 Elliot Shepherd <elliot@identitii.com>
 Enrique Fynn <enriquefynn@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
 Fabian Vogelsteller <fabian@frozeman.de>
 Fabio Berger <fabioberger1991@gmail.com>
 Felix Lange <fjl@twurst.com>
+Frank Wang <eternnoir@gmail.com>
+Gary Rong <garyrong0905@gmail.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guillaume Nicolas <guin56@gmail.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
 Henning Diedrich <hd@eonblast.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
 Jae Kwon <jkwon.work@gmail.com>
+Jamie Pitts <james.pitts@gmail.com>
 Jason Carver <jacarver@linkedin.com>
 Jeff R. Allen <jra@nella.org>
 Jeffrey Wilcke <jeffrey@ethereum.org>
@@ -28,15 +34,20 @@ Jens Agerberg <github@agerberg.me>
 Jonathan Brown <jbrown@bluedroplet.com>
 Joseph Chow <ethereum@outlook.com>
 Justin Clark-Casey <justincc@justincc.org>
+Justin Drake <drakefjustin@gmail.com>
 Kenji Siu <kenji@isuntv.com>
 Kobi Gurkan <kobigurk@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
+Lewis Marshall <lewis@lmars.net>
+Louis Holbrook <dev@holbrook.no>
+Luca Zeug <luclu@users.noreply.github.com>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
 Martin Holst Swende <martin@swende.se>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Micah Zoltu <micah@zoltu.net>
 Nchinda Nchinda <nchinda2@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
@@ -47,17 +58,28 @@ RJ Catalano <rj@erisindustries.com>
 Ramesh Nair <ram@hiddentao.com>
 Ricardo Catalinas Jiménez <r@untroubled.be>
 Rémy Roy <remyroy@remyroy.com>
+Shintaro Kaneko <kaneshin0120@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
 Steven Roose <stevenroose@gmail.com>
 Taylor Gerring <taylor.gerring@gmail.com>
 Thomas Bocek <tom@tomp2p.net>
 Tosh Camille <tochecamille@gmail.com>
+Valentin Wüstholz <wuestholz@users.noreply.github.com>
+Victor Farazdagi <simple.square@gmail.com>
+Victor Tran <vu.tran54@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
 Vincent G <caktux@gmail.com>
 Vitalik Buterin <v@buterin.com>
+Vivek Anand <vivekanand1101@users.noreply.github.com>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
+Zahoor Mohamed <zahoor@zahoor.in>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
+holisticode <holistic.computing@gmail.com>
+ken10100147 <sunhongping@kanjian.com>
+ligi <ligi@ligi.de>
+xiekeyang <xiekeyang@users.noreply.github.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
+Максим Чусовлянов <mchusovlianov@gmail.com>

Dockerfile

@@ -4,11 +4,12 @@ ADD . /go-expanse
 RUN \
   apk add --update git go make gcc musl-dev linux-headers && \
   (cd go-expanse && make gexp)                           && \
-  cp go-expanse/build/bin/gexp /gexp                     && \
+  cp go-expanse/build/bin/gexp /usr/local/bin/           && \
   apk del git go make gcc musl-dev linux-headers          && \
   rm -rf /go-expanse && rm -rf /var/cache/apk/*
 
 EXPOSE 9656
 EXPOSE 42786
+EXPOSE 42786/udp
 
-ENTRYPOINT ["/gexp"]
+ENTRYPOINT ["gexp"]

Makefile

@@ -44,10 +44,10 @@ clean:
 # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.
 
 devtools:
-	go get -u golang.org/x/tools/cmd/stringer
+	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
-	go get -u github.com/jteeuwen/go-bindata/go-bindata
+	env GOBIN= go get -u github.com/jteeuwen/go-bindata/go-bindata
-	go get -u github.com/fjl/gencodec
+	env GOBIN= go get -u github.com/fjl/gencodec
-	go install ./cmd/abigen
+	env GOBIN= go install ./cmd/abigen
 
 # Cross Compilation Targets (xgo)
 

README.md

@@ -70,7 +70,7 @@ This command will:
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/expanse-org/wiki/wiki/JavaScript-API)
    as well as Gexp's own [management APIs](https://github.com/expanse-org/go-expanse/wiki/Management-APIs).
    This too is optional and if you leave it out you can always attach to an already running Gexp instance
-   with `gexp --attach`.
+   with `gexp attach`.
 
 ### Full node on the Expanse test network
 
@@ -84,21 +84,24 @@ $ gexp --testnet --fast --cache=512 console
 ```
 
 The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they
-are equially useful on the testnet too. Please see above for their explanations if you've skipped to
+are equally useful on the testnet too. Please see above for their explanations if you've skipped to
 here.
 
 Specifying the `--testnet` flag however will reconfigure your Gexp instance a bit:
 
  * Instead of using the default data directory (`~/.expanse` on Linux for example), Gexp will nest
-   itself one level deeper into a `testnet` subfolder (`~/.expanse/testnet` on Linux).
+   itself one level deeper into a `testnet` subfolder (`~/.expanse/testnet` on Linux). Note, on OSX
- * Instead of connecting the main Expanse network, the client will connect to the test network,
+   and Linux this also means that attaching to a running testnet node requires the use of a custom
+   endpoint since `gexp attach` will try to attach to a production node endpoint by default. E.g.
+   `gexp attach <datadir>/testnet/gexp.ipc`. Windows users are not affected by this.
+ * Instead of connecting the main Ethereum network, the client will connect to the test network,
+>>>>>>> 6171d01b1195abd7ac75044dcd507d4758d83cde
    which uses different P2P bootnodes, different network IDs and genesis states.
 
 *Note: Although there are some internal protective measures to prevent transactions from crossing
-over between the main network and test network (different starting nonces), you should make sure to
+over between the main network and test network, you should make sure to always use separate accounts
-always use separate accounts for play-money and real-money. Unless you manually move accounts, Gexp
+for play-money and real-money. Unless you manually move accounts, Gexp will by default correctly
-will by default correctly separate the two networks and will not make any accounts available between
+separate the two networks and will not make any accounts available between them.*
-them.*
 
 #### Docker quick start
 
@@ -162,6 +165,12 @@ and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`)
 
 ```json
 {
+  "config": {
+        "chainId": 0,
+        "homesteadBlock": 0,
+        "eip155Block": 0,
+        "eip158Block": 0
+    },
   "alloc"      : {},
   "coinbase"   : "0x0000000000000000000000000000000000000000",
   "difficulty" : "0x20000",

VERSION

@@ -1 +1 @@
-1.6.0
+1.6.6

accounts/abi/bind/backends/simulated.go

@@ -27,6 +27,7 @@ import (
 	"github.com/expanse-org/go-expanse/accounts/abi/bind"
 	"github.com/expanse-org/go-expanse/common"
 	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/core"
 	"github.com/expanse-org/go-expanse/core/state"
 	"github.com/expanse-org/go-expanse/core/types"
@@ -34,7 +35,6 @@ import (
 	"github.com/expanse-org/go-expanse/ethdb"
 	"github.com/expanse-org/go-expanse/event"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 )
 
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
@@ -61,7 +61,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
 	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, genesis.Config, new(pow.FakePow), new(event.TypeMux), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
 	backend.rollback()
 	return backend
@@ -248,7 +248,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	// Execute the call.
 	msg := callmsg{call}
 
-	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain)
+	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
 	vmenv := vm.NewEVM(evmContext, statedb, b.config, vm.Config{})

accounts/keystore/account_cache.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/account_cache_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/key.go

@@ -124,14 +124,13 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) {
 	if err != nil {
 		return err
 	}
-
+	privkey, err := crypto.HexToECDSA(keyJSON.PrivateKey)
-	privkey, err := hex.DecodeString(keyJSON.PrivateKey)
 	if err != nil {
 		return err
 	}
 
 	k.Address = common.BytesToAddress(addr)
-	k.PrivateKey = crypto.ToECDSA(privkey)
+	k.PrivateKey = privkey
 
 	return nil
 }

accounts/keystore/keystore.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -450,7 +450,6 @@ func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (acco
 	if ks.cache.hasAddress(key.Address) {
 		return accounts.Account{}, fmt.Errorf("account already exists")
 	}
-
 	return ks.importKey(key, passphrase)
 }
 

accounts/keystore/keystore_passphrase.go

@@ -182,7 +182,8 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	if err != nil {
 		return nil, err
 	}
-	key := crypto.ToECDSA(keyBytes)
+	key := crypto.ToECDSAUnsafe(keyBytes)
+
 	return &Key{
 		Id:         uuid.UUID(keyId),
 		Address:    crypto.PubkeyToAddress(key.PublicKey),

accounts/keystore/keystore_passphrase_test.go

@@ -46,7 +46,7 @@ func TestKeyEncryptDecrypt(t *testing.T) {
 		// Decrypt with the correct password
 		key, err := DecryptKey(keyjson, password)
 		if err != nil {
-			t.Errorf("test %d: json key failed to decrypt: %v", i, err)
+			t.Fatalf("test %d: json key failed to decrypt: %v", i, err)
 		}
 		if key.Address != address {
 			t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address)

accounts/keystore/keystore_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/keystore/presale.go

@@ -74,7 +74,8 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 		return nil, err
 	}
 	ethPriv := crypto.Keccak256(plainText)
-	ecKey := crypto.ToECDSA(ethPriv)
+	ecKey := crypto.ToECDSAUnsafe(ethPriv)
+
 	key = &Key{
 		Id:         nil,
 		Address:    crypto.PubkeyToAddress(ecKey.PublicKey),

appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.3.windows-%GETH_ARCH%.zip
-  - 7z x go1.8.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.8.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci-notes.md

@@ -5,9 +5,9 @@ for Ubuntu. Packages are built for the all Ubuntu versions which are supported b
 Canonical:
 
 - Trusty Tahr (14.04 LTS)
-- Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
 - Yakkety Yak (16.10)
+- Zesty Zapus (17.04)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,18 +21,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-expanse with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+Add the gophers PPA and install Go 1.8 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.8 devscripts debhelper
 
 Create the source packages:
 
@@ -40,10 +40,11 @@ Create the source packages:
 
 Then go into the source package directory for your running distribution and build the package:
 
-    $ cd dist/expanse-unstable-1.5.0+xenial
+    $ cd dist/expanse-beta-1.5.0+xenial
+
     $ dpkg-buildpackage
 
 Built packages are placed in the dist/ directory.
 
     $ cd ..
-    $ dpkg-deb -c gexp-unstable_1.5.0+xenial_amd64.deb
+    $ dpkg-deb -c gexp-beta_1.6.0+xenial_amd64.deb

build/ci.go

@@ -32,6 +32,7 @@ Available commands are:
    aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
    xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
    xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -73,42 +74,52 @@ var (
 		executablePath("bootnode"),
 		executablePath("evm"),
 		executablePath("gexp"),
-		executablePath("swarm"),
+		executablePath("puppeth"),
 		executablePath("rlpdump"),
+		executablePath("swarm"),
+		executablePath("wnode"),
 	}
 
 // A debian package is created for all executables listed here.
 	debExecutables = []debExecutable{
 		{
-			Name:        "gexp",
+			Name:        "abigen",
-			Description: "Expanse CLI client.",
+			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
 		},
 		{
 			Name:        "bootnode",
 			Description: "Ethereum bootnode.",
 		},
-		{
-			Name:        "rlpdump",
-			Description: "Developer utility tool that prints RLP structures.",
-		},
 		{
 			Name:        "evm",
 			Description: "Developer utility version of the EVM (Expanse Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
+		{
+			Name:        "gexp",
+			Description: "Ethereum CLI client.",
+		},
+		{
+			Name:        "puppeth",
+			Description: "Expanse private network manager.",
+		},
+		{
+			Name:        "rlpdump",
+			Description: "Developer utility tool that prints RLP structures.",
+		},
 		{
 			Name:        "swarm",
 			Description: "Expanse Swarm daemon and tools",
 		},
 		{
-			Name:        "abigen",
+			Name:        "wnode",
-			Description: "Source code generator to convert Expanse contract definitions into easy to use, compile-time type-safe Go packages.",
+			Description: "Ethereum Whisper diagnostic tool",
 		},
 	}
 
-// Distros for which packages are created.
+	// Distros for which packages are created.
-// Note: vivid is unsupported because there is no golang-1.6 package for it.
+	// Note: vivid is unsupported because there is no golang-1.6 package for it.
-// Note: wily is unsupported because it was officially deprecated on lanchpad.
+	// Note: wily is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "yakkety"}
+	debDistros = []string{"trusty", "xenial", "yakkety", "zesty"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -146,6 +157,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "purge":
+		doPurge(os.Args[2:])
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -284,7 +297,8 @@ func doTest(cmdline []string) {
 	// Run analysis tools before the tests.
 	build.MustRun(goTool("vet", packages...))
 	if *misspell {
-		spellcheck(packages)
+		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
+		// spellcheck(packages)
 	}
 	// Run the actual tests.
 	gotest := goTool("test", buildFlags(env)...)
@@ -427,6 +441,10 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a PR build")
 		os.Exit(0)
 	}
+	if env.IsCronJob {
+		log.Printf("skipping because this is a cron job")
+		os.Exit(0)
+	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
@@ -952,3 +970,62 @@ func xgoTool(args []string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// Binary distribution cleanups
+
+func doPurge(cmdline []string) {
+	var (
+		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	if env := build.Env(); !env.IsCronJob {
+		log.Printf("skipping because not a cron job")
+		os.Exit(0)
+	}
+	// Create the azure authentication and list the current archives
+	auth := build.AzureBlobstoreConfig{
+		Account:   strings.Split(*store, "/")[0],
+		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+		Container: strings.SplitN(*store, "/", 2)[1],
+	}
+	blobs, err := build.AzureBlobstoreList(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Iterate over the blobs, collect and sort all unstable builds
+	for i := 0; i < len(blobs); i++ {
+		if !strings.Contains(blobs[i].Name, "unstable") {
+			blobs = append(blobs[:i], blobs[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(blobs); i++ {
+		for j := i + 1; j < len(blobs); j++ {
+			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if iTime.After(jTime) {
+				blobs[i], blobs[j] = blobs[j], blobs[i]
+			}
+		}
+	}
+	// Filter out all archives more recent that the given threshold
+	for i, blob := range blobs {
+		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
+			blobs = blobs[:i]
+			break
+		}
+	}
+	// Delete all marked as such and return
+	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
+		log.Fatal(err)
+	}
+}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.7
+Build-Depends: debhelper (>= 8.0.0), golang-1.8
 Standards-Version: 3.9.5
 Homepage: https://expanse.tech
 Vcs-Git: git://github.com/expanse-org/go-expanse.git
@@ -13,7 +13,7 @@ Architecture: any
 Depends: ${misc:Depends}, {{.ExeList}}
 Description: Meta-package to install gexp and other tools
  Meta-package to install gexp and other tools
- 
+
 {{range .Executables}}
 Package: {{$.ExeName .}}
 Conflicts: {{$.ExeConflicts .}}

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/update-license.go

@@ -47,13 +47,18 @@ var (
 		// boring stuff
 		"vendor/", "tests/files/", "build/",
 		// don't relicense vendored sources
-		"crypto/sha3/", "crypto/ecies/", "log/",
+		"cmd/internal/browser",
+		"consensus/ethash/xor.go",
+		"crypto/bn256/",
+		"crypto/ecies/",
 		"crypto/secp256k1/curve.go",
+		"crypto/sha3/",
+		"internal/jsre/deps",
+		"log/",
 		// don't license generated files
 		"contracts/chequebook/contract/",
 		"contracts/ens/contract/",
 		"contracts/release/contract.go",
-		"p2p/discv5/nodeevent_string.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
@@ -284,6 +289,9 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 		if !stat.Mode().IsRegular() {
 			continue
 		}
+		if isGenerated(file) {
+			continue
+		}
 		info, err := fileInfo(file)
 		if err != nil {
 			fmt.Printf("ERROR %s: %v\n", file, err)
@@ -294,6 +302,23 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
+func isGenerated(file string) bool {
+	fd, err := os.Open(file)
+	if err != nil {
+		return false
+	}
+	defer fd.Close()
+	buf := make([]byte, 2048)
+	n, _ := fd.Read(buf)
+	buf = buf[:n]
+	for _, l := range bytes.Split(buf, []byte("\n")) {
+		if bytes.HasPrefix(l, []byte("// Code generated")) {
+			return true
+		}
+	}
+	return false
+}
+
 // fileInfo finds the lowest year in which the given file was committed.
 func fileInfo(file string) (*info, error) {
 	info := &info{file: file, Year: int64(time.Now().Year())}

cmd/bootnode/main.go

@@ -68,6 +68,7 @@ func main() {
 		if err = crypto.SaveECDSA(*genKey, nodeKey); err != nil {
 			utils.Fatalf("%v", err)
 		}
+		return
 	case *nodeKeyFile == "" && *nodeKeyHex == "":
 		utils.Fatalf("Use -nodekey or -nodekeyhex to specify a private key")
 	case *nodeKeyFile != "" && *nodeKeyHex != "":

cmd/evm/disasm.go

@@ -20,10 +20,10 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"strings"
 
 	"github.com/expanse-org/go-expanse/core/asm"
 	cli "gopkg.in/urfave/cli.v1"
-	"strings"
 )
 
 var disasmCommand = cli.Command{

cmd/evm/main.go

@@ -35,6 +35,18 @@ var (
 		Name:  "debug",
 		Usage: "output full trace logs",
 	}
+	MemProfileFlag = cli.StringFlag{
+		Name:  "memprofile",
+		Usage: "creates a memory profile at the given path",
+	}
+	CPUProfileFlag = cli.StringFlag{
+		Name:  "cpuprofile",
+		Usage: "creates a CPU profile at the given path",
+	}
+	StatDumpFlag = cli.BoolFlag{
+		Name:  "statdump",
+		Usage: "displays stack and heap memory information",
+	}
 	CodeFlag = cli.StringFlag{
 		Name:  "code",
 		Usage: "EVM code",
@@ -93,6 +105,9 @@ func init() {
 		DumpFlag,
 		InputFlag,
 		DisableGasMeteringFlag,
+		MemProfileFlag,
+		CPUProfileFlag,
+		StatDumpFlag,
 	}
 	app.Commands = []cli.Command{
 		compileCommand,

cmd/evm/runner.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"runtime/pprof"
 	"time"
 
 	goruntime "runtime"
@@ -108,6 +109,19 @@ func runCmd(ctx *cli.Context) error {
 		},
 	}
 
+	if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" {
+		f, err := os.Create(cpuProfilePath)
+		if err != nil {
+			fmt.Println("could not create CPU profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			fmt.Println("could not start CPU profile: ", err)
+			os.Exit(1)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
 	tstart := time.Now()
 	if ctx.GlobalBool(CreateFlag.Name) {
 		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
@@ -125,12 +139,27 @@ func runCmd(ctx *cli.Context) error {
 		fmt.Println(string(statedb.Dump()))
 	}
 
+	if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {
+		f, err := os.Create(memProfilePath)
+		if err != nil {
+			fmt.Println("could not create memory profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			fmt.Println("could not write memory profile: ", err)
+			os.Exit(1)
+		}
+		f.Close()
+	}
+
 	if ctx.GlobalBool(DebugFlag.Name) {
 		fmt.Fprintln(os.Stderr, "#### TRACE ####")
 		vm.WriteTrace(os.Stderr, logger.StructLogs())
 		fmt.Fprintln(os.Stderr, "#### LOGS ####")
 		vm.WriteLogs(os.Stderr, statedb.Logs())
+	}
 
+	if ctx.GlobalBool(StatDumpFlag.Name) {
 		var mem goruntime.MemStats
 		goruntime.ReadMemStats(&mem)
 		fmt.Fprintf(os.Stderr, `evm execution time: %v

cmd/faucet/faucet.go

@@ -0,0 +1,543 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// faucet is a Ether faucet backed by a light client.
+package main
+
+//go:generate go-bindata -nometadata -o website.go faucet.html
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"math"
+	"math/big"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts"
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/eth"
+	"github.com/expanse-org/go-expanse/eth/downloader"
+	"github.com/expanse-org/go-expanse/ethclient"
+	"github.com/expanse-org/go-expanse/ethstats"
+	"github.com/expanse-org/go-expanse/les"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/node"
+	"github.com/expanse-org/go-expanse/p2p"
+	"github.com/expanse-org/go-expanse/p2p/discover"
+	"github.com/expanse-org/go-expanse/p2p/discv5"
+	"github.com/expanse-org/go-expanse/p2p/nat"
+	"github.com/expanse-org/go-expanse/params"
+	"golang.org/x/net/websocket"
+)
+
+var (
+	genesisFlag = flag.String("genesis", "", "Genesis json file to seed the chain with")
+	apiPortFlag = flag.Int("apiport", 8080, "Listener port for the HTTP API connection")
+	ethPortFlag = flag.Int("ethport", 30303, "Listener port for the devp2p connection")
+	bootFlag    = flag.String("bootnodes", "", "Comma separated bootnode enode URLs to seed with")
+	netFlag     = flag.Uint64("network", 0, "Network ID to use for the Ethereum protocol")
+	statsFlag   = flag.String("ethstats", "", "Ethstats network monitoring auth string")
+
+	netnameFlag = flag.String("faucet.name", "", "Network name to assign to the faucet")
+	payoutFlag  = flag.Int("faucet.amount", 1, "Number of Ethers to pay out per user request")
+	minutesFlag = flag.Int("faucet.minutes", 1440, "Number of minutes to wait between funding rounds")
+	tiersFlag   = flag.Int("faucet.tiers", 3, "Number of funding tiers to enable (x3 time, x2.5 funds)")
+
+	accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with")
+	accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds")
+
+	githubUser  = flag.String("github.user", "", "GitHub user to authenticate with for Gist access")
+	githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with")
+
+	captchaToken  = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side")
+	captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side")
+
+	logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+)
+
+var (
+	ether = new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
+)
+
+func main() {
+	// Parse the flags and set up the logger to print everything requested
+	flag.Parse()
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*logFlag), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+
+	// Construct the payout tiers
+	amounts := make([]string, *tiersFlag)
+	periods := make([]string, *tiersFlag)
+	for i := 0; i < *tiersFlag; i++ {
+		// Calculate the amount for the next tier and format it
+		amount := float64(*payoutFlag) * math.Pow(2.5, float64(i))
+		amounts[i] = fmt.Sprintf("%s Ethers", strconv.FormatFloat(amount, 'f', -1, 64))
+		if amount == 1 {
+			amounts[i] = strings.TrimSuffix(amounts[i], "s")
+		}
+		// Calculate the period for the next tier and format it
+		period := *minutesFlag * int(math.Pow(3, float64(i)))
+		periods[i] = fmt.Sprintf("%d mins", period)
+		if period%60 == 0 {
+			period /= 60
+			periods[i] = fmt.Sprintf("%d hours", period)
+
+			if period%24 == 0 {
+				period /= 24
+				periods[i] = fmt.Sprintf("%d days", period)
+			}
+		}
+		if period == 1 {
+			periods[i] = strings.TrimSuffix(periods[i], "s")
+		}
+	}
+	// Load up and render the faucet website
+	tmpl, err := Asset("faucet.html")
+	if err != nil {
+		log.Crit("Failed to load the faucet template", "err", err)
+	}
+	website := new(bytes.Buffer)
+	err = template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{
+		"Network":   *netnameFlag,
+		"Amounts":   amounts,
+		"Periods":   periods,
+		"Recaptcha": *captchaToken,
+	})
+	if err != nil {
+		log.Crit("Failed to render the faucet template", "err", err)
+	}
+	// Load and parse the genesis block requested by the user
+	blob, err := ioutil.ReadFile(*genesisFlag)
+	if err != nil {
+		log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err)
+	}
+	genesis := new(core.Genesis)
+	if err = json.Unmarshal(blob, genesis); err != nil {
+		log.Crit("Failed to parse genesis block json", "err", err)
+	}
+	// Convert the bootnodes to internal enode representations
+	var enodes []*discv5.Node
+	for _, boot := range strings.Split(*bootFlag, ",") {
+		if url, err := discv5.ParseNode(boot); err == nil {
+			enodes = append(enodes, url)
+		} else {
+			log.Error("Failed to parse bootnode URL", "url", boot, "err", err)
+		}
+	}
+	// Load up the account key and decrypt its password
+	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
+		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
+	}
+	pass := string(blob)
+
+	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
+	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
+		log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err)
+	}
+	acc, err := ks.Import(blob, pass, pass)
+	if err != nil {
+		log.Crit("Failed to import faucet signer account", "err", err)
+	}
+	ks.Unlock(acc, pass)
+
+	// Assemble and start the faucet light service
+	faucet, err := newFaucet(genesis, *ethPortFlag, enodes, *netFlag, *statsFlag, ks, website.Bytes())
+	if err != nil {
+		log.Crit("Failed to start faucet", "err", err)
+	}
+	defer faucet.close()
+
+	if err := faucet.listenAndServe(*apiPortFlag); err != nil {
+		log.Crit("Failed to launch faucet API", "err", err)
+	}
+}
+
+// request represents an accepted funding request.
+type request struct {
+	Username string             `json:"username"` // GitHub user for displaying an avatar
+	Account  common.Address     `json:"account"`  // Ethereum address being funded
+	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
+	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+}
+
+// faucet represents a crypto faucet backed by an Ethereum light client.
+type faucet struct {
+	config *params.ChainConfig // Chain configurations for signing
+	stack  *node.Node          // Ethereum protocol stack
+	client *ethclient.Client   // Client connection to the Ethereum chain
+	index  []byte              // Index page to serve up on the web
+
+	keystore *keystore.KeyStore // Keystore containing the single signer
+	account  accounts.Account   // Account funding user faucet requests
+	nonce    uint64             // Current pending nonce of the faucet
+	price    *big.Int           // Current gas price to issue funds with
+
+	conns    []*websocket.Conn    // Currently live websocket connections
+	timeouts map[string]time.Time // History of users and their funding timeouts
+	reqs     []*request           // Currently pending funding requests
+	update   chan struct{}        // Channel to signal request updates
+
+	lock sync.RWMutex // Lock protecting the faucet's internals
+}
+
+func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
+	// Assemble the raw devp2p protocol stack
+	stack, err := node.New(&node.Config{
+		Name:    "geth",
+		Version: params.Version,
+		DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
+		P2P: p2p.Config{
+			NAT:              nat.Any(),
+			NoDiscovery:      true,
+			DiscoveryV5:      true,
+			ListenAddr:       fmt.Sprintf(":%d", port),
+			DiscoveryV5Addr:  fmt.Sprintf(":%d", port+1),
+			MaxPeers:         25,
+			BootstrapNodesV5: enodes,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// Assemble the Ethereum light client protocol
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		cfg := eth.DefaultConfig
+		cfg.SyncMode = downloader.LightSync
+		cfg.NetworkId = network
+		cfg.Genesis = genesis
+		return les.New(ctx, &cfg)
+	}); err != nil {
+		return nil, err
+	}
+	// Assemble the ethstats monitoring and reporting service'
+	if stats != "" {
+		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			var serv *les.LightEthereum
+			ctx.Service(&serv)
+			return ethstats.New(stats, nil, serv)
+		}); err != nil {
+			return nil, err
+		}
+	}
+	// Boot up the client and ensure it connects to bootnodes
+	if err := stack.Start(); err != nil {
+		return nil, err
+	}
+	for _, boot := range enodes {
+		old, _ := discover.ParseNode(boot.String())
+		stack.Server().AddPeer(old)
+	}
+	// Attach to the client and retrieve and interesting metadatas
+	api, err := stack.Attach()
+	if err != nil {
+		stack.Stop()
+		return nil, err
+	}
+	client := ethclient.NewClient(api)
+
+	return &faucet{
+		config:   genesis.Config,
+		stack:    stack,
+		client:   client,
+		index:    index,
+		keystore: ks,
+		account:  ks.Accounts()[0],
+		timeouts: make(map[string]time.Time),
+		update:   make(chan struct{}, 1),
+	}, nil
+}
+
+// close terminates the Ethereum connection and tears down the faucet.
+func (f *faucet) close() error {
+	return f.stack.Stop()
+}
+
+// listenAndServe registers the HTTP handlers for the faucet and boots it up
+// for service user funding requests.
+func (f *faucet) listenAndServe(port int) error {
+	go f.loop()
+
+	http.HandleFunc("/", f.webHandler)
+	http.Handle("/api", websocket.Handler(f.apiHandler))
+
+	return http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
+}
+
+// webHandler handles all non-api requests, simply flattening and returning the
+// faucet website.
+func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write(f.index)
+}
+
+// apiHandler handles requests for Ether grants and transaction statuses.
+func (f *faucet) apiHandler(conn *websocket.Conn) {
+	// Start tracking the connection and drop at the end
+	f.lock.Lock()
+	f.conns = append(f.conns, conn)
+	f.lock.Unlock()
+
+	defer func() {
+		f.lock.Lock()
+		for i, c := range f.conns {
+			if c == conn {
+				f.conns = append(f.conns[:i], f.conns[i+1:]...)
+				break
+			}
+		}
+		f.lock.Unlock()
+	}()
+	// Send a few initial stats to the client
+	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+	websocket.JSON.Send(conn, map[string]interface{}{
+		"funds":    balance.Div(balance, ether),
+		"funded":   nonce,
+		"peers":    f.stack.Server().PeerCount(),
+		"requests": f.reqs,
+	})
+	// Send the initial block to the client
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	header, err := f.client.HeaderByNumber(ctx, nil)
+	cancel()
+
+	if err != nil {
+		log.Error("Failed to retrieve latest header", "err", err)
+	} else {
+		websocket.JSON.Send(conn, header)
+	}
+	// Keep reading requests from the websocket until the connection breaks
+	for {
+		// Fetch the next funding request and validate against github
+		var msg struct {
+			URL     string `json:"url"`
+			Tier    uint   `json:"tier"`
+			Captcha string `json:"captcha"`
+		}
+		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+			return
+		}
+		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
+			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+			continue
+		}
+		if msg.Tier >= uint(*tiersFlag) {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"})
+			continue
+		}
+		log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier)
+
+		// If captcha verifications are enabled, make sure we're not dealing with a robot
+		if *captchaToken != "" {
+			form := url.Values{}
+			form.Add("secret", *captchaSecret)
+			form.Add("response", msg.Captcha)
+
+			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			var result struct {
+				Success bool            `json:"success"`
+				Errors  json.RawMessage `json:"error-codes"`
+			}
+			err = json.NewDecoder(res.Body).Decode(&result)
+			res.Body.Close()
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			if !result.Success {
+				log.Warn("Captcha verification failed", "err", string(result.Errors))
+				websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"})
+				continue
+			}
+		}
+		// Retrieve the gist from the GitHub Gist APIs
+		parts := strings.Split(msg.URL, "/")
+		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+		if *githubUser != "" {
+			req.SetBasicAuth(*githubUser, *githubToken)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		var gist struct {
+			Owner struct {
+				Login string `json:"login"`
+			} `json:"owner"`
+			Files map[string]struct {
+				Content string `json:"content"`
+			} `json:"files"`
+		}
+		err = json.NewDecoder(res.Body).Decode(&gist)
+		res.Body.Close()
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		if gist.Owner.Login == "" {
+			websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"})
+			continue
+		}
+		// Iterate over all the files and look for Ethereum addresses
+		var address common.Address
+		for _, file := range gist.Files {
+			if len(file.Content) == 2+common.AddressLength*2 {
+				address = common.HexToAddress(file.Content)
+			}
+		}
+		if address == (common.Address{}) {
+			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
+			continue
+		}
+		// Validate the user's existence since the API is unhelpful here
+		if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		res.Body.Close()
+
+		if res.StatusCode != 200 {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"})
+			continue
+		}
+		// Ensure the user didn't request funds too recently
+		f.lock.Lock()
+		var (
+			fund    bool
+			timeout time.Time
+		)
+		if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) {
+			// User wasn't funded recently, create the funding transaction
+			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
+			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
+			amount = new(big.Int).Div(amount, new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(msg.Tier)), nil))
+
+			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil)
+			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			// Submit the transaction and mark as funded if successful
+			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			f.reqs = append(f.reqs, &request{
+				Username: gist.Owner.Login,
+				Account:  address,
+				Time:     time.Now(),
+				Tx:       signed,
+			})
+			f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			fund = true
+		}
+		f.lock.Unlock()
+
+		// Send an error if too frequent funding, othewise a success
+		if !fund {
+			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))})
+			continue
+		}
+		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		select {
+		case f.update <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// loop keeps waiting for interesting events and pushes them out to connected
+// websockets.
+func (f *faucet) loop() {
+	// Wait for chain events and push them to clients
+	heads := make(chan *types.Header, 16)
+	sub, err := f.client.SubscribeNewHead(context.Background(), heads)
+	if err != nil {
+		log.Crit("Failed to subscribe to head events", "err", err)
+	}
+	defer sub.Unsubscribe()
+
+	for {
+		select {
+		case head := <-heads:
+			// New chain head arrived, query the current stats and stream to clients
+			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+			balance = new(big.Int).Div(balance, ether)
+
+			price, _ := f.client.SuggestGasPrice(context.Background())
+			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+			f.lock.Lock()
+			f.price, f.nonce = price, nonce
+			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+				f.reqs = f.reqs[1:]
+			}
+			f.lock.Unlock()
+
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{
+					"funds":    balance,
+					"funded":   f.nonce,
+					"peers":    f.stack.Server().PeerCount(),
+					"requests": f.reqs,
+				}); err != nil {
+					log.Warn("Failed to send stats to client", "err", err)
+					conn.Close()
+					continue
+				}
+				if err := websocket.JSON.Send(conn, head); err != nil {
+					log.Warn("Failed to send header to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+
+		case <-f.update:
+			// Pending requests updated, stream to clients
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+					log.Warn("Failed to send requests to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+		}
+	}
+}

cmd/faucet/faucet.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8">
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+
+		<title>{{.Network}}: GitHub Faucet</title>
+
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
+
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-noty/2.4.1/packaged/jquery.noty.packaged.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.18.0/moment.min.js"></script>
+
+		<style>
+			.vertical-center {
+				min-height: 100%;
+				min-height: 100vh;
+				display: flex;
+				align-items: center;
+			}
+			.progress {
+				position: relative;
+			}
+			.progress span {
+				position: absolute;
+				display: block;
+				width: 100%;
+				color: white;
+			 }
+			 pre {
+				 padding: 6px;
+				 margin: 0;
+			 }
+		</style>
+	</head>
+
+	<body>
+		<div class="vertical-center">
+			<div class="container">
+				<div class="row" style="margin-bottom: 16px;">
+					<div class="col-lg-12">
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+					</div>
+				</div>
+				<div class="row">
+					<div class="col-lg-8 col-lg-offset-2">
+						<div class="input-group">
+							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<span class="input-group-btn">
+								<button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Give me Ether	<i class="fa fa-caret-down" aria-hidden="true"></i></button>
+				        <ul class="dropdown-menu dropdown-menu-right">{{range $idx, $amount := .Amounts}}
+				          <li><a style="text-align: center;" onclick="tier={{$idx}}; {{if $.Recaptcha}}grecaptcha.execute(){{else}}submit({{$idx}}){{end}}">{{$amount}} / {{index $.Periods $idx}}</a></li>{{end}}
+				        </ul>
+							</span>
+						</div>{{if .Recaptcha}}
+						<div class="g-recaptcha" data-sitekey="{{.Recaptcha}}" data-callback="submit" data-size="invisible"></div>{{end}}
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+						<div class="col-lg-6 col-lg-offset-3">
+						<div class="panel panel-small panel-default">
+							<div class="panel-body" style="padding: 0; overflow: auto; max-height: 300px;">
+								<table id="requests" class="table table-condensed" style="margin: 0;"></table>
+							</div>
+							<div class="panel-footer">
+								<table style="width: 100%"><tr>
+									<td style="text-align: center;"><i class="fa fa-rss" aria-hidden="true"></i> <span id="peers"></span> peers</td>
+									<td style="text-align: center;"><i class="fa fa-database" aria-hidden="true"></i> <span id="block"></span> blocks</td>
+									<td style="text-align: center;"><i class="fa fa-heartbeat" aria-hidden="true"></i> <span id="funds"></span> Ethers</td>
+									<td style="text-align: center;"><i class="fa fa-university" aria-hidden="true"></i> <span id="funded"></span> funded</td>
+								</tr></table>
+							</div>
+						</div>
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+					<div class="col-lg-12">
+						<h3>How does this work?</h3>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.</p>
+						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+						{{if .Recaptcha}}<em>The faucet is running invisible reCaptcha protection against bots.</em>{{end}}
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<script>
+			// Global variables to hold the current status of the faucet
+			var attempt = 0;
+			var server;
+			var tier = 0;
+
+			// Define the function that submits a gist url to the server
+			var submit = function({{if .Recaptcha}}captcha{{end}}) {
+				server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
+				grecaptcha.reset();{{end}}
+			};
+			// Define a method to reconnect upon server loss
+			var reconnect = function() {
+				if (attempt % 2 == 0) {
+					server = new WebSocket("wss://" + location.host + "/api");
+				} else {
+					server = new WebSocket("ws://" + location.host + "/api");
+				}
+				attempt++;
+
+				server.onmessage = function(event) {
+					var msg = JSON.parse(event.data);
+					if (msg === null) {
+						return;
+					}
+
+					if (msg.funds !== undefined) {
+						$("#funds").text(msg.funds);
+					}
+					if (msg.funded !== undefined) {
+						$("#funded").text(msg.funded);
+					}
+					if (msg.peers !== undefined) {
+						$("#peers").text(msg.peers);
+					}
+					if (msg.number !== undefined) {
+						$("#block").text(parseInt(msg.number, 16));
+					}
+					if (msg.error !== undefined) {
+						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+					}
+					if (msg.success !== undefined) {
+						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+					}
+					if (msg.requests !== undefined && msg.requests !== null) {
+						var content = "";
+						for (var i=0; i<msg.requests.length; i++) {
+							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+						}
+						$("#requests").html("<tbody>" + content + "</tbody>");
+					}
+				}
+				server.onclose = function() { setTimeout(reconnect, 3000); };
+			}
+			// Establish a websocket connection to the API server
+			reconnect();
+		</script>{{if .Recaptcha}}
+		<script src="https://www.google.com/recaptcha/api.js" async defer></script>{{end}}
+	</body>
+</html>

cmd/gexp/accountcmd.go

@@ -40,32 +40,39 @@ var (
 
 will prompt for your password and imports your ether presale account.
 It can be used non-interactively with the --password option taking a
-passwordfile as argument containing the wallet password in plaintext.
+passwordfile as argument containing the wallet password in plaintext.`,
-
-`,
 		Subcommands: []cli.Command{
 			{
-				Action:    importWallet,
+
 				Name:      "import",
 				Usage:     "Import Expanse presale wallet",
 				ArgsUsage: "<keyFile>",
+				Action:    utils.MigrateFlags(importWallet),
+				Category:  "ACCOUNT COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
-TODO: Please write this
+	geth wallet [options] /path/to/my/presale.wallet
-`,
+
+will prompt for your password and imports your ether presale account.
+It can be used non-interactively with the --password option taking a
+passwordfile as argument containing the wallet password in plaintext.`,
 			},
 		},
 	}
-	accountCommand = cli.Command{
-		Action:    accountList,
-		Name:      "account",
-		Usage:     "Manage accounts",
-		ArgsUsage: "",
-		Category:  "ACCOUNT COMMANDS",
-		Description: `
-Manage accounts lets you create new accounts, list all existing accounts,
-import a private key into a new account.
 
-'            help' shows a list of subcommands or help for one subcommand.
+	accountCommand = cli.Command{
+		Name:     "account",
+		Usage:    "Manage accounts",
+		Category: "ACCOUNT COMMANDS",
+		Description: `
+
+Manage accounts, list all existing accounts, import a private key into a new
+account, create a new account or update an existing account.
 
 It supports interactive mode, when you are prompted for password as well as
 non-interactive mode where passwords are supplied via a given password file.
@@ -80,36 +87,34 @@ Note that exporting your key in unencrypted format is NOT supported.
 Keys are stored under <DATADIR>/keystore.
 It is safe to transfer the entire directory or the individual keys therein
 between expanse nodes by simply copying.
-Make sure you backup your keys regularly.
 
-In order to use your account to send transactions, you need to unlock them using
+Make sure you backup your keys regularly.`,
-the '--unlock' option. The argument is a space separated list of addresses or
-indexes. If used non-interactively with a passwordfile, the file should contain
-the respective passwords one per line. If you unlock n accounts and the password
-file contains less than n entries, then the last password is meant to apply to
-all remaining accounts.
-
-And finally. DO NOT FORGET YOUR PASSWORD.
-`,
 		Subcommands: []cli.Command{
 			{
-				Action:    accountList,
+				Name:   "list",
-				Name:      "list",
+				Usage:  "Print summary of existing accounts",
-				Usage:     "Print account addresses",
+				Action: utils.MigrateFlags(accountList),
-				ArgsUsage: " ",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+				},
 				Description: `
-TODO: Please write this
+Print a short summary of all accounts`,
-`,
 			},
 			{
-				Action:    accountCreate,
+				Name:   "new",
-				Name:      "new",
+				Usage:  "Create a new account",
-				Usage:     "Create a new account",
+				Action: utils.MigrateFlags(accountCreate),
-				ArgsUsage: " ",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     gexp account new
 
-Creates a new account. Prints the address.
+Creates a new account and prints the address.
 
 The account is saved in encrypted format, you are prompted for a passphrase.
 
@@ -117,17 +122,20 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    gexp --password <passwordfile> account new
-
 Note, this is meant to be used for testing only, it is a bad idea to save your
 password to file or expose in any other way.
 `,
 			},
 			{
-				Action:    accountUpdate,
 				Name:      "update",
 				Usage:     "Update an existing account",
+				Action:    utils.MigrateFlags(accountUpdate),
 				ArgsUsage: "<address>",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     gexp account update <address>
 
@@ -141,16 +149,22 @@ format to the newest format or change the password for an account.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    gexp --password <passwordfile> account update <address>
+    gexp account update [options] <address>
 
 Since only one password can be given, only format update can be performed,
 changing your password is only possible interactively.
 `,
 			},
 			{
-				Action:    accountImport,
+				Name:   "import",
-				Name:      "import",
+				Usage:  "Import a private key into a new account",
-				Usage:     "Import a private key into a new account",
+				Action: utils.MigrateFlags(accountImport),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				ArgsUsage: "<keyFile>",
 				Description: `
     gexp account import <keyfile>
@@ -166,7 +180,7 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the -password flag:
 
-    gexp --password <passwordfile> account import <keyfile>
+    gexp account import [options] <keyfile>
 
 Note:
 As you can directly copy your encrypted accounts to another expanse instance,
@@ -179,8 +193,7 @@ nodes.
 )
 
 func accountList(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
-
 	var index int
 	for _, wallet := range stack.AccountManager().Wallets() {
 		for _, account := range wallet.Accounts() {
@@ -278,7 +291,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -296,13 +309,15 @@ func accountUpdate(ctx *cli.Context) error {
 	if len(ctx.Args()) == 0 {
 		utils.Fatalf("No accounts specified to update")
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	account, oldPassword := unlockAccount(ctx, ks, ctx.Args().First(), 0, nil)
+	for _, addr := range ctx.Args() {
-	newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
+		account, oldPassword := unlockAccount(ctx, ks, addr, 0, nil)
-	if err := ks.Update(account, oldPassword, newPassword); err != nil {
+		newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
-		utils.Fatalf("Could not update the account: %v", err)
+		if err := ks.Update(account, oldPassword, newPassword); err != nil {
+			utils.Fatalf("Could not update the account: %v", err)
+		}
 	}
 	return nil
 }
@@ -317,7 +332,7 @@ func importWallet(ctx *cli.Context) error {
 		utils.Fatalf("Could not read wallet file: %v", err)
 	}
 
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -338,7 +353,7 @@ func accountImport(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("Failed to load the private key: %v", err)
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)

cmd/gexp/accountcmd_test.go

@@ -43,13 +43,13 @@ func tmpDatadirWithKeystore(t *testing.T) string {
 }
 
 func TestAccountListEmpty(t *testing.T) {
-	gexp := runGeth(t, "account")
+	gexp := runGeth(t, "account", "list")
 	gexp.expectExit()
 }
 
 func TestAccountList(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	gexp := runGeth(t, "--datadir", datadir, "account")
+	gexp := runGeth(t, "account", "list", "--datadir", datadir)
 	defer gexp.expectExit()
 	if runtime.GOOS == "windows" {
 		gexp.expect(`
@@ -67,7 +67,7 @@ Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/k
 }
 
 func TestAccountNew(t *testing.T) {
-	gexp := runGeth(t, "--lightkdf", "account", "new")
+	gexp := runGeth(t, "account", "new", "--lightkdf")
 	defer gexp.expectExit()
 	gexp.expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
@@ -79,7 +79,7 @@ Repeat passphrase: {{.InputLine "foobar"}}
 }
 
 func TestAccountNewBadRepeat(t *testing.T) {
-	gexp := runGeth(t, "--lightkdf", "account", "new")
+	gexp := runGeth(t, "account", "new", "--lightkdf")
 	defer gexp.expectExit()
 	gexp.expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
@@ -92,9 +92,9 @@ Fatal: Passphrases do not match
 
 func TestAccountUpdate(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	gexp := runGeth(t,
+	gexp := runGeth(t, "account", "update",
 		"--datadir", datadir, "--lightkdf",
-		"account", "update", "f466859ead1932d743d622cb74fc058882e8648a")
+		"f466859ead1932d743d622cb74fc058882e8648a")
 	defer gexp.expectExit()
 	gexp.expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
@@ -107,7 +107,7 @@ Repeat passphrase: {{.InputLine "foobar2"}}
 }
 
 func TestWalletImport(t *testing.T) {
-	gexp := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
+	gexp := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
 	defer gexp.expectExit()
 	gexp.expect(`
 !! Unsupported terminal, password will be echoed.
@@ -122,7 +122,8 @@ Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
 }
 
 func TestWalletImportBadPassword(t *testing.T) {
-	gexp := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
+
+	gexp := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
 	defer gexp.expectExit()
 	gexp.expect(`
 !! Unsupported terminal, password will be echoed.

cmd/gexp/bugcmd.go

@@ -29,11 +29,12 @@ import (
 	"github.com/expanse-org/go-expanse/cmd/internal/browser"
 	"github.com/expanse-org/go-expanse/params"
 
+	"github.com/expanse-org/go-expanse/cmd/utils"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
 var bugCommand = cli.Command{
-	Action:    reportBug,
+	Action:    utils.MigrateFlags(reportBug),
 	Name:      "bug",
 	Usage:     "opens a window to report a bug on the gexp repo",
 	ArgsUsage: " ",

cmd/gexp/chaincmd.go

@@ -40,80 +40,98 @@ import (
 
 var (
 	initCommand = cli.Command{
-		Action:    initGenesis,
+		Action:    utils.MigrateFlags(initGenesis),
 		Name:      "init",
 		Usage:     "Bootstrap and initialize a new genesis block",
 		ArgsUsage: "<genesisPath>",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
 participating.
-`,
+
+It expects the genesis file as argument.`,
 	}
 	importCommand = cli.Command{
-		Action:    importChain,
+		Action:    utils.MigrateFlags(importChain),
 		Name:      "import",
 		Usage:     "Import a blockchain file",
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-The import command imports blocks from an RLP-encoded form. The form can be one file 
+The import command imports blocks from an RLP-encoded form. The form can be one file
-with several RLP-encoded blocks, or several files can be used. 
+with several RLP-encoded blocks, or several files can be used.
-If only one file is used, import error will result in failure. If several files are used, 
+
-processing will proceed even if an individual RLP-file import failure occurs.   
+If only one file is used, import error will result in failure. If several files are used,
-`,
+processing will proceed even if an individual RLP-file import failure occurs.`,
 	}
 	exportCommand = cli.Command{
-		Action:    exportChain,
+		Action:    utils.MigrateFlags(exportChain),
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
-if already existing.
+if already existing.`,
-`,
 	}
 	removedbCommand = cli.Command{
-		Action:    removeDB,
+		Action:    utils.MigrateFlags(removeDB),
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: " ",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-TODO: Please write this
+Remove blockchain and state databases`,
-`,
 	}
 	dumpCommand = cli.Command{
-		Action:    dump,
+		Action:    utils.MigrateFlags(dump),
 		Name:      "dump",
 		Usage:     "Dump a specific block from storage",
 		ArgsUsage: "[<blockHash> | <blockNum>]...",
-		Category:  "BLOCKCHAIN COMMANDS",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The arguments are interpreted as block numbers or hashes.
-Use "expanse dump 0" to dump the genesis block.
+Use "expanse dump 0" to dump the genesis block.`,
-`,
 	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
 // the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
 func initGenesis(ctx *cli.Context) error {
+	// Make sure we have a valid genesis JSON
 	genesisPath := ctx.Args().First()
 	if len(genesisPath) == 0 {
-		utils.Fatalf("must supply path to genesis JSON file")
+		utils.Fatalf("Must supply path to genesis JSON file")
 	}
-
-	stack := makeFullNode(ctx)
-	chaindb := utils.MakeChainDatabase(ctx, stack)
-
 	file, err := os.Open(genesisPath)
 	if err != nil {
-		utils.Fatalf("failed to read genesis file: %v", err)
+		utils.Fatalf("Failed to read genesis file: %v", err)
 	}
 	defer file.Close()
 
@@ -121,12 +139,19 @@ func initGenesis(ctx *cli.Context) error {
 	if err := json.NewDecoder(file).Decode(genesis); err != nil {
 		utils.Fatalf("invalid genesis file: %v", err)
 	}
-
+	// Open an initialise both full and light databases
-	_, hash, err := core.SetupGenesisBlock(chaindb, genesis)
+	stack := makeFullNode(ctx)
-	if err != nil {
+	for _, name := range []string{"chaindata", "lightchaindata"} {
-		utils.Fatalf("failed to write genesis block: %v", err)
+		chaindb, err := stack.OpenDatabase(name, 0, 0)
+		if err != nil {
+			utils.Fatalf("Failed to open database: %v", err)
+		}
+		_, hash, err := core.SetupGenesisBlock(chaindb, genesis)
+		if err != nil {
+			utils.Fatalf("Failed to write genesis block: %v", err)
+		}
+		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
-	log.Info("Successfully wrote genesis state", "hash", hash)
 	return nil
 }
 
@@ -244,25 +269,30 @@ func exportChain(ctx *cli.Context) error {
 }
 
 func removeDB(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
-	dbdir := stack.ResolvePath(utils.ChainDbName(ctx))
-	if !common.FileExist(dbdir) {
-		fmt.Println(dbdir, "does not exist")
-		return nil
-	}
 
-	fmt.Println(dbdir)
+	for _, name := range []string{"chaindata", "lightchaindata"} {
-	confirm, err := console.Stdin.PromptConfirm("Remove this database?")
+		// Ensure the database exists in the first place
-	switch {
+		logger := log.New("database", name)
-	case err != nil:
+
-		utils.Fatalf("%v", err)
+		dbdir := stack.ResolvePath(name)
-	case !confirm:
+		if !common.FileExist(dbdir) {
-		fmt.Println("Operation aborted")
+			logger.Info("Database doesn't exist, skipping", "path", dbdir)
-	default:
+			continue
-		fmt.Println("Removing...")
+		}
-		start := time.Now()
+		// Confirm removal and execute
-		os.RemoveAll(dbdir)
+		fmt.Println(dbdir)
-		fmt.Printf("Removed in %v\n", time.Since(start))
+		confirm, err := console.Stdin.PromptConfirm("Remove this database?")
+		switch {
+		case err != nil:
+			utils.Fatalf("%v", err)
+		case !confirm:
+			logger.Warn("Database deletion aborted")
+		default:
+			start := time.Now()
+			os.RemoveAll(dbdir)
+			logger.Info("Database successfully deleted", "elapsed", common.PrettyDuration(time.Since(start)))
+		}
 	}
 	return nil
 }

cmd/gexp/config.go

@@ -0,0 +1,187 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"unicode"
+
+	cli "gopkg.in/urfave/cli.v1"
+
+	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/contracts/release"
+	"github.com/expanse-org/go-expanse/eth"
+	"github.com/expanse-org/go-expanse/node"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/naoina/toml"
+)
+
+var (
+	dumpConfigCommand = cli.Command{
+		Action:      utils.MigrateFlags(dumpConfig),
+		Name:        "dumpconfig",
+		Usage:       "Show configuration values",
+		ArgsUsage:   "",
+		Flags:       append(nodeFlags, rpcFlags...),
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: `The dumpconfig command shows configuration values.`,
+	}
+
+	configFileFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "TOML configuration file",
+	}
+)
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}
+
+type ethstatsConfig struct {
+	URL string `toml:",omitempty"`
+}
+
+type gethConfig struct {
+	Eth      eth.Config
+	Node     node.Config
+	Ethstats ethstatsConfig
+}
+
+func loadConfig(file string, cfg *gethConfig) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(cfg)
+	// Add file name to errors that have a line number.
+	if _, ok := err.(*toml.LineError); ok {
+		err = errors.New(file + ", " + err.Error())
+	}
+	return err
+}
+
+func defaultNodeConfig() node.Config {
+	cfg := node.DefaultConfig
+	cfg.Name = clientIdentifier
+	cfg.Version = params.VersionWithCommit(gitCommit)
+	cfg.HTTPModules = append(cfg.HTTPModules, "eth, exp")
+	cfg.WSModules = append(cfg.WSModules, "eth, exp")
+	cfg.IPCPath = "gexp.ipc"
+	return cfg
+}
+
+func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
+	// Load defaults.
+	cfg := gethConfig{
+		Eth:  eth.DefaultConfig,
+		Node: defaultNodeConfig(),
+	}
+
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+
+	// Apply flags.
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	stack, err := node.New(&cfg.Node)
+	if err != nil {
+		utils.Fatalf("Failed to create the protocol stack: %v", err)
+	}
+	utils.SetEthConfig(ctx, stack, &cfg.Eth)
+	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
+		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
+	}
+
+	return stack, cfg
+}
+
+func makeFullNode(ctx *cli.Context) *node.Node {
+	stack, cfg := makeConfigNode(ctx)
+
+	utils.RegisterEthService(stack, &cfg.Eth)
+
+	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
+	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	if shhEnabled || shhAutoEnabled {
+		utils.RegisterShhService(stack)
+	}
+
+	// Add the Ethereum Stats daemon if requested.
+	if cfg.Ethstats.URL != "" {
+		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
+	}
+
+	// Add the release oracle service so it boots along with node.
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		config := release.Config{
+			Oracle: relOracle,
+			Major:  uint32(params.VersionMajor),
+			Minor:  uint32(params.VersionMinor),
+			Patch:  uint32(params.VersionPatch),
+		}
+		commit, _ := hex.DecodeString(gitCommit)
+		copy(config.Commit[:], commit)
+		return release.NewReleaseService(ctx, config)
+	}); err != nil {
+		utils.Fatalf("Failed to register the Gexp release oracle service: %v", err)
+	}
+	return stack
+}
+
+// dumpConfig is the dumpconfig command.
+func dumpConfig(ctx *cli.Context) error {
+	_, cfg := makeConfigNode(ctx)
+	comment := ""
+
+	if cfg.Eth.Genesis != nil {
+		cfg.Eth.Genesis = nil
+		comment += "# Note: this config doesn't contain the genesis block.\n\n"
+	}
+
+	out, err := tomlSettings.Marshal(&cfg)
+	if err != nil {
+		return err
+	}
+	io.WriteString(os.Stdout, comment)
+	os.Stdout.Write(out)
+	return nil
+}

cmd/gexp/consolecmd.go

@@ -29,23 +29,26 @@ import (
 )
 
 var (
+	consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag}
+
 	consoleCommand = cli.Command{
-		Action:    localConsole,
+		Action:   utils.MigrateFlags(localConsole),
-		Name:      "console",
+		Name:     "console",
-		Usage:     "Start an interactive JavaScript environment",
+		Usage:    "Start an interactive JavaScript environment",
-		ArgsUsage: "", // TODO: Write this!
+		Flags:    append(append(nodeFlags, rpcFlags...), consoleFlags...),
-		Category:  "CONSOLE COMMANDS",
+		Category: "CONSOLE COMMANDS",
 		Description: `
 The Gexp console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console
+See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console.`,
-`,
 	}
+
 	attachCommand = cli.Command{
-		Action:    remoteConsole,
+		Action:    utils.MigrateFlags(remoteConsole),
 		Name:      "attach",
 		Usage:     "Start an interactive JavaScript environment (connect to node)",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "[endpoint]",
+		Flags:     append(consoleFlags, utils.DataDirFlag),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The Gexp console is an interactive shell for the JavaScript runtime environment
@@ -54,11 +57,13 @@ See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console.
 This command allows to open a console on a running gexp node.
 `,
 	}
+
 	javascriptCommand = cli.Command{
-		Action:    ephemeralConsole,
+		Action:    utils.MigrateFlags(ephemeralConsole),
 		Name:      "js",
 		Usage:     "Execute the specified JavaScript files",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "<jsfile> [jsfile...]",
+		Flags:     append(nodeFlags, consoleFlags...),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
@@ -81,11 +86,12 @@ func localConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc gexp: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
@@ -118,17 +124,18 @@ func remoteConsole(ctx *cli.Context) error {
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
 	}
 	defer console.Stop(false)
 
-	// If only a short execution was requested, evaluate and return
 	if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" {
 		console.Evaluate(script)
 		return nil
 	}
+
 	// Otherwise print the welcome screen and enter interactive mode
 	console.Welcome()
 	console.Interactive()
@@ -151,7 +158,7 @@ func dialRPC(endpoint string) (*rpc.Client, error) {
 }
 
 // ephemeralConsole starts a new gexp node, attaches an ephemeral JavaScript
-// console to it, and each of the files specified as arguments and tears the
+// console to it, executes each of the files specified as arguments and tears
 // everything down.
 func ephemeralConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
@@ -165,11 +172,12 @@ func ephemeralConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc gexp: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)

cmd/gexp/consolecmd_test.go

@@ -22,14 +22,17 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/rpc"
+)
+
+const (
+	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
+	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
 // Tests that a node embedded within a console can be started up properly and
@@ -45,24 +48,21 @@ func TestConsoleWelcome(t *testing.T) {
 
 	// Gather all the infos the welcome message needs to contain
 	gexp.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	gexp.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	gexp.setTemplateFunc("gover", runtime.Version)
 	gexp.setTemplateFunc("gethver", func() string { return params.Version })
 	gexp.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
-	gexp.setTemplateFunc("apis", func() []string {
+	gexp.setTemplateFunc("apis", func() string { return ipcAPIs })
-		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		sort.Strings(apis)
-		return apis
-	})
 
 	// Verify the actual welcome message to the required template
 	gexp.expect(`
 Welcome to the Gexp JavaScript console!
 
-instance: Gexp/v{{gethver}}/{{goos}}/{{gover}}
+instance: Gexp/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{.Etherbase}}
 at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit"}}
 `)
@@ -88,7 +88,7 @@ func TestIPCAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "ipc:"+ipc)
+	testAttachWelcome(t, gexp, "ipc:"+ipc, ipcAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
@@ -102,7 +102,7 @@ func TestHTTPAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--rpc", "--rpcport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "http://localhost:"+port)
+	testAttachWelcome(t, gexp, "http://localhost:"+port, httpAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
@@ -117,13 +117,13 @@ func TestWSAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--ws", "--wsport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, gexp, "ws://localhost:"+port)
+	testAttachWelcome(t, gexp, "ws://localhost:"+port, httpAPIs)
 
 	gexp.interrupt()
 	gexp.expectExit()
 }
 
-func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint string) {
+func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint, apis string) {
 	// Attach to a running gexp note and terminate immediately
 	attach := runGeth(t, "attach", endpoint)
 	defer attach.expectExit()
@@ -131,32 +131,24 @@ func testAttachWelcome(t *testing.T, gexp *testgeth, endpoint string) {
 
 	// Gather all the infos the welcome message needs to contain
 	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
+	attach.setTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	attach.setTemplateFunc("gover", runtime.Version)
 	attach.setTemplateFunc("gethver", func() string { return params.Version })
 	attach.setTemplateFunc("etherbase", func() string { return gexp.Etherbase })
 	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
 	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
 	attach.setTemplateFunc("datadir", func() string { return gexp.Datadir })
-	attach.setTemplateFunc("apis", func() []string {
+	attach.setTemplateFunc("apis", func() string { return apis })
-		var apis []string
-		if strings.HasPrefix(endpoint, "ipc") {
-			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		} else {
-			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
-		}
-		sort.Strings(apis)
-		return apis
-	})
 
 	// Verify the actual welcome message to the required template
 	attach.expect(`
 Welcome to the Gexp JavaScript console!
 
-instance: Gexp/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{etherbase}}
 at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit" }}
 `)

cmd/gexp/dao_test.go

@@ -84,27 +84,24 @@ var daoGenesisForkBlock = big.NewInt(314)
 // set in the database after various initialization procedures and invocations.
 func TestDAOForkBlockNewChain(t *testing.T) {
 	for i, arg := range []struct {
-		testnet     bool
 		genesis     string
 		expectBlock *big.Int
 		expectVote  bool
 	}{
 		// Test DAO Default Mainnet
-		{false, "", params.MainNetDAOForkBlock, true},
+		{"", params.MainNetDAOForkBlock, true},
-		// test DAO Default Testnet
-		{true, "", params.TestNetDAOForkBlock, true},
 		// test DAO Init Old Privnet
-		{false, daoOldGenesis, nil, false},
+		{daoOldGenesis, nil, false},
 		// test DAO Default No Fork Privnet
-		{false, daoNoForkGenesis, daoGenesisForkBlock, false},
+		{daoNoForkGenesis, daoGenesisForkBlock, false},
 		// test DAO Default Pro Fork Privnet
-		{false, daoProForkGenesis, daoGenesisForkBlock, true},
+		{daoProForkGenesis, daoGenesisForkBlock, true},
 	} {
-		testDAOForkBlockNewChain(t, i, arg.testnet, arg.genesis, arg.expectBlock, arg.expectVote)
+		testDAOForkBlockNewChain(t, i, arg.genesis, arg.expectBlock, arg.expectVote)
 	}
 }
 
-func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis string, expectBlock *big.Int, expectVote bool) {
+func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
 	datadir := tmpdir(t)
 	defer os.RemoveAll(datadir)
@@ -119,17 +116,11 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	} else {
 		// Force chain initialization
 		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
-		if testnet {
-			args = append(args, "--testnet")
-		}
 		gexp := runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...)
 		gexp.cmd.Wait()
 	}
 	// Retrieve the DAO config flag from the database
-	path := filepath.Join(datadir, "gexp", "chaindata")
+	path := filepath.Join(datadir, "geth", "chaindata")
-	if testnet && genesis == "" {
-		path = filepath.Join(datadir, "testnet", "gexp", "chaindata")
-	}
 	db, err := ethdb.NewLDBDatabase(path, 0, 0)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
@@ -137,9 +128,6 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	defer db.Close()
 
 	genesisHash := common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
-	if testnet {
-		genesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
-	}
 	if genesis != "" {
 		genesisHash = daoGenesisHash
 	}

cmd/gexp/main.go

@@ -18,7 +18,6 @@
 package main
 
 import (
-	"encoding/hex"
 	"fmt"
 	"os"
 	"runtime"
@@ -29,17 +28,13 @@ import (
 	"github.com/expanse-org/go-expanse/accounts/keystore"
 	"github.com/expanse-org/go-expanse/cmd/utils"
 	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/common/hexutil"
 	"github.com/expanse-org/go-expanse/console"
-	"github.com/expanse-org/go-expanse/contracts/release"
 	"github.com/expanse-org/go-expanse/eth"
 	"github.com/expanse-org/go-expanse/ethclient"
 	"github.com/expanse-org/go-expanse/internal/debug"
 	"github.com/expanse-org/go-expanse/log"
 	"github.com/expanse-org/go-expanse/metrics"
 	"github.com/expanse-org/go-expanse/node"
-	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/rlp"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -54,13 +49,89 @@ var (
 	relOracle = common.HexToAddress("0x926d69cc3bbf81d52cba6886d788df007a15a3cd")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-expanse command line interface")
+	// flags that configure the node
+	nodeFlags = []cli.Flag{
+		utils.IdentityFlag,
+		utils.UnlockedAccountFlag,
+		utils.PasswordFileFlag,
+		utils.BootnodesFlag,
+		utils.BootnodesV4Flag,
+		utils.BootnodesV5Flag,
+		utils.DataDirFlag,
+		utils.KeyStoreDirFlag,
+		utils.NoUSBFlag,
+		utils.EthashCacheDirFlag,
+		utils.EthashCachesInMemoryFlag,
+		utils.EthashCachesOnDiskFlag,
+		utils.EthashDatasetDirFlag,
+		utils.EthashDatasetsInMemoryFlag,
+		utils.EthashDatasetsOnDiskFlag,
+		utils.TxPoolPriceLimitFlag,
+		utils.TxPoolPriceBumpFlag,
+		utils.TxPoolAccountSlotsFlag,
+		utils.TxPoolGlobalSlotsFlag,
+		utils.TxPoolAccountQueueFlag,
+		utils.TxPoolGlobalQueueFlag,
+		utils.TxPoolLifetimeFlag,
+		utils.FastSyncFlag,
+		utils.LightModeFlag,
+		utils.SyncModeFlag,
+		utils.LightServFlag,
+		utils.LightPeersFlag,
+		utils.LightKDFFlag,
+		utils.CacheFlag,
+		utils.TrieCacheGenFlag,
+		utils.ListenPortFlag,
+		utils.MaxPeersFlag,
+		utils.MaxPendingPeersFlag,
+		utils.EtherbaseFlag,
+		utils.GasPriceFlag,
+		utils.MinerThreadsFlag,
+		utils.MiningEnabledFlag,
+		utils.TargetGasLimitFlag,
+		utils.NATFlag,
+		utils.NoDiscoverFlag,
+		utils.DiscoveryV5Flag,
+		utils.NetrestrictFlag,
+		utils.NodeKeyFileFlag,
+		utils.NodeKeyHexFlag,
+		utils.WhisperEnabledFlag,
+		utils.DevModeFlag,
+		utils.TestnetFlag,
+		utils.RinkebyFlag,
+		utils.VMEnableDebugFlag,
+		utils.NetworkIdFlag,
+		utils.RPCCORSDomainFlag,
+		utils.EthStatsURLFlag,
+		utils.MetricsEnabledFlag,
+		utils.FakePoWFlag,
+		utils.NoCompactionFlag,
+		utils.GpoBlocksFlag,
+		utils.GpoPercentileFlag,
+		utils.ExtraDataFlag,
+		configFileFlag,
+	}
+
+	rpcFlags = []cli.Flag{
+		utils.RPCEnabledFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCPortFlag,
+		utils.RPCApiFlag,
+		utils.WSEnabledFlag,
+		utils.WSListenAddrFlag,
+		utils.WSPortFlag,
+		utils.WSApiFlag,
+		utils.WSAllowedOriginsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+	}
 )
 
 func init() {
-	// Initialize the CLI app and start Gexp
+	// Initialize the CLI app and start Geth
 	app.Action = gexp
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2017 The go-ethereum / go-expanse Authors"
+	app.Copyright = "Copyright 2013-2017 The go-ethereum Authors & the go-expanse Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
@@ -82,79 +153,13 @@ func init() {
 		versionCommand,
 		bugCommand,
 		licenseCommand,
+		// See config.go
+		dumpConfigCommand,
 	}
 
-	app.Flags = []cli.Flag{
+	app.Flags = append(app.Flags, nodeFlags...)
-		utils.IdentityFlag,
+	app.Flags = append(app.Flags, rpcFlags...)
-		utils.UnlockedAccountFlag,
+	app.Flags = append(app.Flags, consoleFlags...)
-		utils.PasswordFileFlag,
-		utils.BootnodesFlag,
-		utils.DataDirFlag,
-		utils.KeyStoreDirFlag,
-		utils.EthashCacheDirFlag,
-		utils.EthashCachesInMemoryFlag,
-		utils.EthashCachesOnDiskFlag,
-		utils.EthashDatasetDirFlag,
-		utils.EthashDatasetsInMemoryFlag,
-		utils.EthashDatasetsOnDiskFlag,
-		utils.FastSyncFlag,
-		utils.LightModeFlag,
-		utils.LightServFlag,
-		utils.LightPeersFlag,
-		utils.LightKDFFlag,
-		utils.CacheFlag,
-		utils.TrieCacheGenFlag,
-		utils.JSpathFlag,
-		utils.ListenPortFlag,
-		utils.MaxPeersFlag,
-		utils.MaxPendingPeersFlag,
-		utils.EtherbaseFlag,
-		utils.GasPriceFlag,
-		utils.MinerThreadsFlag,
-		utils.MiningEnabledFlag,
-		utils.TargetGasLimitFlag,
-		utils.NATFlag,
-		utils.NoDiscoverFlag,
-		utils.DiscoveryV5Flag,
-		utils.NetrestrictFlag,
-		utils.NodeKeyFileFlag,
-		utils.NodeKeyHexFlag,
-		utils.RPCEnabledFlag,
-		utils.RPCListenAddrFlag,
-		utils.RPCPortFlag,
-		utils.RPCApiFlag,
-		utils.WSEnabledFlag,
-		utils.WSListenAddrFlag,
-		utils.WSPortFlag,
-		utils.WSApiFlag,
-		utils.WSAllowedOriginsFlag,
-		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
-		utils.IPCPathFlag,
-		utils.ExecFlag,
-		utils.PreloadJSFlag,
-		utils.WhisperEnabledFlag,
-		utils.DevModeFlag,
-		utils.TestNetFlag,
-		utils.VMForceJitFlag,
-		utils.VMJitCacheFlag,
-		utils.VMEnableJitFlag,
-		utils.VMEnableDebugFlag,
-		utils.NetworkIdFlag,
-		utils.RPCCORSDomainFlag,
-		utils.EthStatsURLFlag,
-		utils.MetricsEnabledFlag,
-		utils.FakePoWFlag,
-		utils.NoCompactionFlag,
-		utils.SolcPathFlag,
-		utils.GpoMinGasPriceFlag,
-		utils.GpoMaxGasPriceFlag,
-		utils.GpoFullBlockRatioFlag,
-		utils.GpobaseStepDownFlag,
-		utils.GpobaseStepUpFlag,
-		utils.GpobaseCorrectionFactorFlag,
-		utils.ExtraDataFlag,
-	}
 	app.Flags = append(app.Flags, debug.Flags...)
 
 	app.Before = func(ctx *cli.Context) error {
@@ -165,12 +170,6 @@ func init() {
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		// This should be the only place where reporting is enabled
-		// because it is not intended to run while testing.
-		// In addition to this check, bad block reports are sent only
-		// for chains with the main network genesis block and network id 1.
-		eth.EnableBadBlockReporting = false
-
 		utils.SetupNetwork(ctx)
 		return nil
 	}
@@ -199,52 +198,6 @@ func gexp(ctx *cli.Context) error {
 	return nil
 }
 
-func makeFullNode(ctx *cli.Context) *node.Node {
-	// Create the default extradata and construct the base node
-	var clientInfo = struct {
-		Version   uint
-		Name      string
-		GoVersion string
-		Os        string
-	}{uint(params.VersionMajor<<16 | params.VersionMinor<<8 | params.VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
-	extra, err := rlp.EncodeToBytes(clientInfo)
-	if err != nil {
-		log.Warn("Failed to set canonical miner information", "err", err)
-	}
-	if uint64(len(extra)) > params.MaximumExtraDataSize {
-		log.Warn("Miner extra data exceed limit", "extra", hexutil.Bytes(extra), "limit", params.MaximumExtraDataSize)
-		extra = nil
-	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	utils.RegisterEthService(ctx, stack, extra)
-
-	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
-	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
-	if shhEnabled || shhAutoEnabled {
-		utils.RegisterShhService(stack)
-	}
-	// Add the Ethereum Stats daemon if requested
-	if url := ctx.GlobalString(utils.EthStatsURLFlag.Name); url != "" {
-		utils.RegisterEthStatsService(stack, url)
-	}
-	// Add the release oracle service so it boots along with node.
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		config := release.Config{
-			Oracle: relOracle,
-			Major:  uint32(params.VersionMajor),
-			Minor:  uint32(params.VersionMinor),
-			Patch:  uint32(params.VersionPatch),
-		}
-		commit, _ := hex.DecodeString(gitCommit)
-		copy(config.Commit[:], commit)
-		return release.NewReleaseService(ctx, config)
-	}); err != nil {
-		utils.Fatalf("Failed to register the Gexp release oracle service: %v", err)
-	}
-	return stack
-}
-
 // startNode boots up the system node and all registered protocols, after which
 // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the
 // miner.
@@ -303,7 +256,15 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("expanse service not running: %v", err)
 		}
-		if err := ethereum.StartMining(ctx.GlobalInt(utils.MinerThreadsFlag.Name)); err != nil {
+		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
+			type threaded interface {
+				SetThreads(threads int)
+			}
+			if th, ok := ethereum.Engine().(threaded); ok {
+				th.SetThreads(threads)
+			}
+		}
+		if err := ethereum.StartMining(true); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/gexp/misccmd.go

@@ -26,15 +26,15 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/consensus/ethash"
 	"github.com/expanse-org/go-expanse/eth"
 	"github.com/expanse-org/go-expanse/params"
-	"github.com/expanse-org/go-expanse/pow"
 	"gopkg.in/urfave/cli.v1"
 )
 
 var (
 	makedagCommand = cli.Command{
-		Action:    makedag,
+		Action:    utils.MigrateFlags(makedag),
 		Name:      "makedag",
 		Usage:     "Generate ethash DAG (for testing)",
 		ArgsUsage: "<blockNum> <outputDir>",
@@ -47,7 +47,7 @@ Regular users do not need to execute it.
 `,
 	}
 	versionCommand = cli.Command{
-		Action:    version,
+		Action:    utils.MigrateFlags(version),
 		Name:      "version",
 		Usage:     "Print version numbers",
 		ArgsUsage: " ",
@@ -57,7 +57,7 @@ The output of this command is supposed to be machine-readable.
 `,
 	}
 	licenseCommand = cli.Command{
-		Action:    license,
+		Action:    utils.MigrateFlags(license),
 		Name:      "license",
 		Usage:     "Display license information",
 		ArgsUsage: " ",
@@ -87,7 +87,7 @@ func makedag(ctx *cli.Context) error {
 				utils.Fatalf("Can't find dir")
 			}
 			fmt.Println("making DAG, this could take awhile...")
-			pow.MakeDataset(blockNum, dir)
+			ethash.MakeDataset(blockNum, dir)
 		}
 	default:
 		wrongArgs()
@@ -101,10 +101,11 @@ func version(ctx *cli.Context) error {
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
+	fmt.Println("Architecture:", runtime.GOARCH)
 	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
-	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
+	fmt.Println("Network Id:", eth.DefaultConfig.NetworkId)
 	fmt.Println("Go Version:", runtime.Version())
-	fmt.Println("OS:", runtime.GOOS)
+	fmt.Println("Operating System:", runtime.GOOS)
 	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
 	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
 	return nil

cmd/gexp/monitorcmd.go

@@ -49,7 +49,7 @@ var (
 		Usage: "Refresh interval in seconds",
 	}
 	monitorCommand = cli.Command{
-		Action:    monitor,
+		Action:    utils.MigrateFlags(monitor), // keep track of migration progress
 		Name:      "monitor",
 		Usage:     "Monitor and visualize node metrics",
 		ArgsUsage: " ",

cmd/gexp/usage.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"io"
+	"sort"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
 	"github.com/expanse-org/go-expanse/internal/debug"
@@ -30,7 +31,7 @@ import (
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2016 The go-ethereum / go-expanse Authors
+   Copyright 2013-2017 The go-ethereum / go-expanse Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -64,14 +65,17 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "ETHEREUM",
 		Flags: []cli.Flag{
+			configFileFlag,
 			utils.DataDirFlag,
 			utils.KeyStoreDirFlag,
+			utils.NoUSBFlag,
 			utils.NetworkIdFlag,
-			utils.TestNetFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
 			utils.DevModeFlag,
+			utils.SyncModeFlag,
+			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
-			utils.FastSyncFlag,
-			utils.LightModeFlag,
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
@@ -88,6 +92,18 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.EthashDatasetsOnDiskFlag,
 		},
 	},
+	{
+		Name: "TRANSACTION POOL",
+		Flags: []cli.Flag{
+			utils.TxPoolPriceLimitFlag,
+			utils.TxPoolPriceBumpFlag,
+			utils.TxPoolAccountSlotsFlag,
+			utils.TxPoolGlobalSlotsFlag,
+			utils.TxPoolAccountQueueFlag,
+			utils.TxPoolGlobalQueueFlag,
+			utils.TxPoolLifetimeFlag,
+		},
+	},
 	{
 		Name: "PERFORMANCE TUNING",
 		Flags: []cli.Flag{
@@ -115,7 +131,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.WSApiFlag,
 			utils.WSAllowedOriginsFlag,
 			utils.IPCDisabledFlag,
-			utils.IPCApiFlag,
 			utils.IPCPathFlag,
 			utils.RPCCORSDomainFlag,
 			utils.JSpathFlag,
@@ -127,12 +142,15 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "NETWORKING",
 		Flags: []cli.Flag{
 			utils.BootnodesFlag,
+			utils.BootnodesV4Flag,
+			utils.BootnodesV5Flag,
 			utils.ListenPortFlag,
 			utils.MaxPeersFlag,
 			utils.MaxPendingPeersFlag,
 			utils.NATFlag,
 			utils.NoDiscoverFlag,
 			utils.DiscoveryV5Flag,
+			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
 		},
@@ -151,43 +169,70 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "GAS PRICE ORACLE",
 		Flags: []cli.Flag{
-			utils.GpoMinGasPriceFlag,
+			utils.GpoBlocksFlag,
-			utils.GpoMaxGasPriceFlag,
+			utils.GpoPercentileFlag,
-			utils.GpoFullBlockRatioFlag,
-			utils.GpobaseStepDownFlag,
-			utils.GpobaseStepUpFlag,
-			utils.GpobaseCorrectionFactorFlag,
 		},
 	},
 	{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
-			utils.VMEnableJitFlag,
-			utils.VMForceJitFlag,
-			utils.VMJitCacheFlag,
 			utils.VMEnableDebugFlag,
 		},
 	},
 	{
 		Name: "LOGGING AND DEBUGGING",
 		Flags: append([]cli.Flag{
-			utils.EthStatsURLFlag,
 			utils.MetricsEnabledFlag,
 			utils.FakePoWFlag,
+			utils.NoCompactionFlag,
 		}, debug.Flags...),
 	},
+	{
+		Name: "DEPRECATED",
+		Flags: []cli.Flag{
+			utils.FastSyncFlag,
+			utils.LightModeFlag,
+		},
+	},
 	{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
 		},
 	},
-	{
+}
-		Name: "MISCELLANEOUS",
+
-		Flags: []cli.Flag{
+// byCategory sorts an array of flagGroup by Name in the order
-			utils.SolcPathFlag,
+// defined in AppHelpFlagGroups.
-		},
+type byCategory []flagGroup
-	},
+
+func (a byCategory) Len() int      { return len(a) }
+func (a byCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byCategory) Less(i, j int) bool {
+	iCat, jCat := a[i].Name, a[j].Name
+	iIdx, jIdx := len(AppHelpFlagGroups), len(AppHelpFlagGroups) // ensure non categorized flags come last
+
+	for i, group := range AppHelpFlagGroups {
+		if iCat == group.Name {
+			iIdx = i
+		}
+		if jCat == group.Name {
+			jIdx = i
+		}
+	}
+
+	return iIdx < jIdx
+}
+
+func flagCategory(flag cli.Flag) string {
+	for _, category := range AppHelpFlagGroups {
+		for _, flg := range category.Flags {
+			if flg.GetName() == flag.GetName() {
+				return category.Name
+			}
+		}
+	}
+	return "MISC"
 }
 
 func init() {
@@ -199,6 +244,7 @@ func init() {
 		App        interface{}
 		FlagGroups []flagGroup
 	}
+
 	// Override the default app help printer, but only for the global app help
 	originalHelpPrinter := cli.HelpPrinter
 	cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
@@ -228,6 +274,27 @@ func init() {
 			}
 			// Render out custom usage screen
 			originalHelpPrinter(w, tmpl, helpData{data, AppHelpFlagGroups})
+		} else if tmpl == utils.CommandHelpTemplate {
+			// Iterate over all command specific flags and categorize them
+			categorized := make(map[string][]cli.Flag)
+			for _, flag := range data.(cli.Command).Flags {
+				if _, ok := categorized[flag.String()]; !ok {
+					categorized[flagCategory(flag)] = append(categorized[flagCategory(flag)], flag)
+				}
+			}
+
+			// sort to get a stable ordering
+			sorted := make([]flagGroup, 0, len(categorized))
+			for cat, flgs := range categorized {
+				sorted = append(sorted, flagGroup{cat, flgs})
+			}
+			sort.Sort(byCategory(sorted))
+
+			// add sorted array to data and render with default printer
+			originalHelpPrinter(w, tmpl, map[string]interface{}{
+				"cmd":              data,
+				"categorizedFlags": sorted,
+			})
 		} else {
 			originalHelpPrinter(w, tmpl, data)
 		}

cmd/puppeth/module.go

@@ -0,0 +1,152 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+var (
+	// ErrServiceUnknown is returned when a service container doesn't exist.
+	ErrServiceUnknown = errors.New("service unknown")
+
+	// ErrServiceOffline is returned when a service container exists, but it is not
+	// running.
+	ErrServiceOffline = errors.New("service offline")
+
+	// ErrServiceUnreachable is returned when a service container is running, but
+	// seems to not respond to communication attempts.
+	ErrServiceUnreachable = errors.New("service unreachable")
+
+	// ErrNotExposed is returned if a web-service doesn't have an exposed port, nor
+	// a reverse-proxy in front of it to forward requests.
+	ErrNotExposed = errors.New("service not exposed, nor proxied")
+)
+
+// containerInfos is a heavily reduced version of the huge inspection dataset
+// returned from docker inspect, parsed into a form easily usable by puppeth.
+type containerInfos struct {
+	running bool              // Flag whether the container is running currently
+	envvars map[string]string // Collection of environmental variables set on the container
+	portmap map[string]int    // Port mapping from internal port/proto combos to host binds
+	volumes map[string]string // Volume mount points from container to host directories
+}
+
+// inspectContainer runs docker inspect against a running container
+func inspectContainer(client *sshClient, container string) (*containerInfos, error) {
+	// Check whether there's a container running for the service
+	out, err := client.Run(fmt.Sprintf("docker inspect %s", container))
+	if err != nil {
+		return nil, ErrServiceUnknown
+	}
+	// If yes, extract various configuration options
+	type inspection struct {
+		State struct {
+			Running bool
+		}
+		Mounts []struct {
+			Source      string
+			Destination string
+		}
+		Config struct {
+			Env []string
+		}
+		HostConfig struct {
+			PortBindings map[string][]map[string]string
+		}
+	}
+	var inspects []inspection
+	if err = json.Unmarshal(out, &inspects); err != nil {
+		return nil, err
+	}
+	inspect := inspects[0]
+
+	// Infos retrieved, parse the above into something meaningful
+	infos := &containerInfos{
+		running: inspect.State.Running,
+		envvars: make(map[string]string),
+		portmap: make(map[string]int),
+		volumes: make(map[string]string),
+	}
+	for _, envvar := range inspect.Config.Env {
+		if parts := strings.Split(envvar, "="); len(parts) == 2 {
+			infos.envvars[parts[0]] = parts[1]
+		}
+	}
+	for portname, details := range inspect.HostConfig.PortBindings {
+		if len(details) > 0 {
+			port, _ := strconv.Atoi(details[0]["HostPort"])
+			infos.portmap[portname] = port
+		}
+	}
+	for _, mount := range inspect.Mounts {
+		infos.volumes[mount.Destination] = mount.Source
+	}
+	return infos, err
+}
+
+// tearDown connects to a remote machine via SSH and terminates docker containers
+// running with the specified name in the specified network.
+func tearDown(client *sshClient, network string, service string, purge bool) ([]byte, error) {
+	// Tear down the running (or paused) container
+	out, err := client.Run(fmt.Sprintf("docker rm -f %s_%s_1", network, service))
+	if err != nil {
+		return out, err
+	}
+	// If requested, purge the associated docker image too
+	if purge {
+		return client.Run(fmt.Sprintf("docker rmi %s/%s", network, service))
+	}
+	return nil, nil
+}
+
+// resolve retrieves the hostname a service is running on either by returning the
+// actual server name and port, or preferably an nginx virtual host if available.
+func resolve(client *sshClient, network string, service string, port int) (string, error) {
+	// Inspect the service to get various configurations from it
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, service))
+	if err != nil {
+		return "", err
+	}
+	if !infos.running {
+		return "", ErrServiceOffline
+	}
+	// Container online, extract any environmental variables
+	if vhost := infos.envvars["VIRTUAL_HOST"]; vhost != "" {
+		return vhost, nil
+	}
+	return fmt.Sprintf("%s:%d", client.server, port), nil
+}
+
+// checkPort tries to connect to a remote host on a given
+func checkPort(host string, port int) error {
+	log.Trace("Verifying remote TCP connectivity", "server", host, "port", port)
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), time.Second)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}

cmd/puppeth/module_ethstats.go

@@ -0,0 +1,159 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// ethstatsDockerfile is the Dockerfile required to build an ethstats backend
+// and associated monitoring site.
+var ethstatsDockerfile = `
+FROM mhart/alpine-node:latest
+
+RUN \
+  apk add --update git                                         && \
+  git clone --depth=1 https://github.com/karalabe/eth-netstats && \
+	apk del git && rm -rf /var/cache/apk/*                       && \
+	\
+  cd /eth-netstats && npm install && npm install -g grunt-cli && grunt
+
+WORKDIR /eth-netstats
+EXPOSE 3000
+
+RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: []};' > lib/utils/config.js
+
+CMD ["npm", "start"]
+`
+
+// ethstatsComposefile is the docker-compose.yml file required to deploy and
+// maintain an ethstats monitoring site.
+var ethstatsComposefile = `
+version: '2'
+services:
+  ethstats:
+    build: .
+    image: {{.Network}}/ethstats{{if not .VHost}}
+    ports:
+      - "{{.Port}}:3000"{{end}}
+    environment:
+      - WS_SECRET={{.Secret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}{{end}}
+    restart: always
+`
+
+// deployEthstats deploys a new ethstats container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	for i, address := range trusted {
+		trusted[i] = fmt.Sprintf("\"%s\"", address)
+	}
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"Trusted": strings.Join(trusted, ", "),
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+		"Secret":  secret,
+		"VHost":   vhost,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// ethstatsInfos is returned from an ethstats status check to allow reporting
+// various configuration parameters.
+type ethstatsInfos struct {
+	host   string
+	port   int
+	secret string
+	config string
+}
+
+// String implements the stringer interface.
+func (info *ethstatsInfos) String() string {
+	return fmt.Sprintf("host=%s, port=%d, secret=%s", info.host, info.port, info.secret)
+}
+
+// checkEthstats does a health-check against an ethstats server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkEthstats(client *sshClient, network string) (*ethstatsInfos, error) {
+	// Inspect a possible ethstats container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_ethstats_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["3000/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and configure the connection string
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	secret := infos.envvars["WS_SECRET"]
+	config := fmt.Sprintf("%s@%s", secret, host)
+	if port != 80 && port != 443 {
+		config += fmt.Sprintf(":%d", port)
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Ethstats service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &ethstatsInfos{
+		host:   host,
+		port:   port,
+		secret: secret,
+		config: config,
+	}, nil
+}

cmd/puppeth/module_faucet.go

@@ -0,0 +1,227 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// faucetDockerfile is the Dockerfile required to build an faucet container to
+// grant crypto tokens based on GitHub authentications.
+var faucetDockerfile = `
+FROM alpine:latest
+
+RUN mkdir /go
+ENV GOPATH /go
+
+RUN \
+  apk add --update git go make gcc musl-dev ca-certificates linux-headers                             && \
+	mkdir -p $GOPATH/src/github.com/ethereum                                                            && \
+	(cd $GOPATH/src/github.com/ethereum && git clone --depth=1 https://github.com/expanse-org/go-expanse) && \
+  go build -v github.com/expanse-org/go-expanse/cmd/faucet                                              && \
+  apk del git go make gcc musl-dev linux-headers                                                      && \
+  rm -rf $GOPATH && rm -rf /var/cache/apk/*
+
+ADD genesis.json /genesis.json
+ADD account.json /account.json
+ADD account.pass /account.pass
+
+EXPOSE 8080
+
+CMD [ \
+	"/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", "--ethport", "{{.EthPort}}", \
+	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",          \
+	"--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass"                       \
+	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}                                                        \
+]`
+
+// faucetComposefile is the docker-compose.yml file required to deploy and maintain
+// a crypto faucet.
+var faucetComposefile = `
+version: '2'
+services:
+  faucet:
+    build: .
+    image: {{.Network}}/faucet
+    ports:
+      - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}}
+      - "{{.ApiPort}}:8080"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.faucet
+    environment:
+      - ETH_PORT={{.EthPort}}
+      - ETH_NAME={{.EthName}}
+      - FAUCET_AMOUNT={{.FaucetAmount}}
+      - FAUCET_MINUTES={{.FaucetMinutes}}
+      - FAUCET_TIERS={{.FaucetTiers}}
+      - GITHUB_USER={{.GitHubUser}}
+      - GITHUB_TOKEN={{.GitHubToken}}
+      - CAPTCHA_TOKEN={{.CaptchaToken}}
+      - CAPTCHA_SECRET={{.CaptchaSecret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}
+      - VIRTUAL_PORT=8080{{end}}
+    restart: always
+`
+
+// deployFaucet deploys a new faucet container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployFaucet(client *sshClient, network string, bootnodes []string, config *faucetInfos) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID":     config.node.network,
+		"Bootnodes":     strings.Join(bootnodes, ","),
+		"Ethstats":      config.node.ethstats,
+		"EthPort":       config.node.portFull,
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"CaptchaToken":  config.captchaToken,
+		"CaptchaSecret": config.captchaSecret,
+		"FaucetName":    strings.Title(network),
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+		"FaucetTiers":   config.tiers,
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(faucetComposefile)).Execute(composefile, map[string]interface{}{
+		"Network":       network,
+		"Datadir":       config.node.datadir,
+		"VHost":         config.host,
+		"ApiPort":       config.port,
+		"EthPort":       config.node.portFull,
+		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
+		"GitHubUser":    config.githubUser,
+		"GitHubToken":   config.githubToken,
+		"CaptchaToken":  config.captchaToken,
+		"CaptchaSecret": config.captchaSecret,
+		"FaucetAmount":  config.amount,
+		"FaucetMinutes": config.minutes,
+		"FaucetTiers":   config.tiers,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.node.genesis)
+	files[filepath.Join(workdir, "account.json")] = []byte(config.node.keyJSON)
+	files[filepath.Join(workdir, "account.pass")] = []byte(config.node.keyPass)
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the faucet service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// faucetInfos is returned from an faucet status check to allow reporting various
+// configuration parameters.
+type faucetInfos struct {
+	node          *nodeInfos
+	host          string
+	port          int
+	amount        int
+	minutes       int
+	tiers         int
+	githubUser    string
+	githubToken   string
+	captchaToken  string
+	captchaSecret string
+}
+
+// String implements the stringer interface.
+func (info *faucetInfos) String() string {
+	return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, tiers=%d, github=%s, captcha=%v, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.tiers, info.githubUser, info.captchaToken != "", info.node.ethstats)
+}
+
+// checkFaucet does a health-check against an faucet server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
+	// Inspect a possible faucet container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_faucet_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["8080/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and the config values
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	amount, _ := strconv.Atoi(infos.envvars["FAUCET_AMOUNT"])
+	minutes, _ := strconv.Atoi(infos.envvars["FAUCET_MINUTES"])
+	tiers, _ := strconv.Atoi(infos.envvars["FAUCET_TIERS"])
+
+	// Retrieve the funding account informations
+	var out []byte
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.json", network)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_faucet_1 cat /account.pass", network)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Faucet service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &faucetInfos{
+		node: &nodeInfos{
+			datadir:  infos.volumes["/root/.faucet"],
+			portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			ethstats: infos.envvars["ETH_NAME"],
+			keyJSON:  keyJSON,
+			keyPass:  keyPass,
+		},
+		host:          host,
+		port:          port,
+		amount:        amount,
+		minutes:       minutes,
+		tiers:         tiers,
+		githubUser:    infos.envvars["GITHUB_USER"],
+		githubToken:   infos.envvars["GITHUB_TOKEN"],
+		captchaToken:  infos.envvars["CAPTCHA_TOKEN"],
+		captchaSecret: infos.envvars["CAPTCHA_SECRET"],
+	}, nil
+}

cmd/puppeth/module_nginx.go

@@ -0,0 +1,106 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// nginxDockerfile is theis the Dockerfile required to build an nginx reverse-
+// proxy.
+var nginxDockerfile = `FROM jwilder/nginx-proxy`
+
+// nginxComposefile is the docker-compose.yml file required to deploy and maintain
+// an nginx reverse-proxy. The proxy is responsible for exposing one or more HTTP
+// services running on a single host.
+var nginxComposefile = `
+version: '2'
+services:
+  nginx:
+    build: .
+    image: {{.Network}}/nginx
+    ports:
+      - "{{.Port}}:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    restart: always
+`
+
+// deployNginx deploys a new nginx reverse-proxy container to expose one or more
+// HTTP services running on a single host. If an instance with the specified
+// network name already exists there, it will be overwritten!
+func deployNginx(client *sshClient, network string, port int) ([]byte, error) {
+	log.Info("Deploying nginx reverse-proxy", "server", client.server, "port", port)
+
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxDockerfile)).Execute(dockerfile, nil)
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nginxComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nginxInfos is returned from an nginx reverse-proxy status check to allow
+// reporting various configuration parameters.
+type nginxInfos struct {
+	port int
+}
+
+// String implements the stringer interface.
+func (info *nginxInfos) String() string {
+	return fmt.Sprintf("port=%d", info.port)
+}
+
+// checkNginx does a health-check against an nginx reverse-proxy to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkNginx(client *sshClient, network string) (*nginxInfos, error) {
+	// Inspect a possible nginx container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_nginx_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Container available, assemble and return the useful infos
+	return &nginxInfos{
+		port: infos.portmap["80/tcp"],
+	}, nil
+}

cmd/puppeth/module_node.go

@@ -0,0 +1,237 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// nodeDockerfile is the Dockerfile required to run an Ethereum node.
+var nodeDockerfile = `
+FROM ethereum/client-go:alpine-develop
+
+ADD genesis.json /genesis.json
+{{if .Unlock}}
+	ADD signer.json /signer.json
+	ADD signer.pass /signer.pass
+{{end}}
+RUN \
+  echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
+	echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+
+ENTRYPOINT ["/bin/sh", "geth.sh"]
+`
+
+// nodeComposefile is the docker-compose.yml file required to deploy and maintain
+// an Ethereum node (bootnode or miner for now).
+var nodeComposefile = `
+version: '2'
+services:
+  {{.Type}}:
+    build: .
+    image: {{.Network}}/{{.Type}}
+    ports:
+      - "{{.FullPort}}:{{.FullPort}}"
+      - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
+      - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.ethereum
+    environment:
+      - FULL_PORT={{.FullPort}}/tcp
+      - LIGHT_PORT={{.LightPort}}/udp
+      - TOTAL_PEERS={{.TotalPeers}}
+      - LIGHT_PEERS={{.LightPeers}}
+      - STATS_NAME={{.Ethstats}}
+      - MINER_NAME={{.Etherbase}}
+      - GAS_TARGET={{.GasTarget}}
+      - GAS_PRICE={{.GasPrice}}
+    restart: always
+`
+
+// deployNode deploys a new Ethereum node container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos) ([]byte, error) {
+	kind := "sealnode"
+	if config.keyJSON == "" && config.etherbase == "" {
+		kind = "bootnode"
+		bootv4 = make([]string, 0)
+		bootv5 = make([]string, 0)
+	}
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	lightFlag := ""
+	if config.peersLight > 0 {
+		lightFlag = fmt.Sprintf("--lightpeers=%d --lightserv=50", config.peersLight)
+	}
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NetworkID": config.network,
+		"Port":      config.portFull,
+		"Peers":     config.peersTotal,
+		"LightFlag": lightFlag,
+		"BootV4":    strings.Join(bootv4, ","),
+		"BootV5":    strings.Join(bootv5, ","),
+		"Ethstats":  config.ethstats,
+		"Etherbase": config.etherbase,
+		"GasTarget": uint64(1000000 * config.gasTarget),
+		"GasPrice":  uint64(1000000000 * config.gasPrice),
+		"Unlock":    config.keyJSON != "",
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(nodeComposefile)).Execute(composefile, map[string]interface{}{
+		"Type":       kind,
+		"Datadir":    config.datadir,
+		"Network":    network,
+		"FullPort":   config.portFull,
+		"TotalPeers": config.peersTotal,
+		"Light":      config.peersLight > 0,
+		"LightPort":  config.portFull + 1,
+		"LightPeers": config.peersLight,
+		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
+		"Etherbase":  config.etherbase,
+		"GasTarget":  config.gasTarget,
+		"GasPrice":   config.gasPrice,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	//genesisfile, _ := json.MarshalIndent(config.genesis, "", "  ")
+	files[filepath.Join(workdir, "genesis.json")] = []byte(config.genesis)
+
+	if config.keyJSON != "" {
+		files[filepath.Join(workdir, "signer.json")] = []byte(config.keyJSON)
+		files[filepath.Join(workdir, "signer.pass")] = []byte(config.keyPass)
+	}
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the boot or seal node service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// nodeInfos is returned from a boot or seal node status check to allow reporting
+// various configuration parameters.
+type nodeInfos struct {
+	genesis    []byte
+	network    int64
+	datadir    string
+	ethstats   string
+	portFull   int
+	portLight  int
+	enodeFull  string
+	enodeLight string
+	peersTotal int
+	peersLight int
+	etherbase  string
+	keyJSON    string
+	keyPass    string
+	gasTarget  float64
+	gasPrice   float64
+}
+
+// String implements the stringer interface.
+func (info *nodeInfos) String() string {
+	discv5 := ""
+	if info.peersLight > 0 {
+		discv5 = fmt.Sprintf(", portv5=%d", info.portLight)
+	}
+	return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s, gastarget=%0.3f MGas, gasprice=%0.3f GWei",
+		info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats, info.gasTarget, info.gasPrice)
+}
+
+// checkNode does a health-check against an boot or seal node server to verify
+// whether it's running, and if yes, whether it's responsive.
+func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) {
+	kind := "bootnode"
+	if !boot {
+		kind = "sealnode"
+	}
+	// Inspect a possible bootnode container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, kind))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve a few types from the environmental variables
+	totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
+	lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
+	gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64)
+	gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64)
+
+	// Container available, retrieve its node ID and its genesis json
+	var out []byte
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 /geth --exec admin.nodeInfo.id attach", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
+		return nil, ErrServiceUnreachable
+	}
+	genesis := bytes.TrimSpace(out)
+
+	keyJSON, keyPass := "", ""
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.json", network, kind)); err == nil {
+		keyJSON = string(bytes.TrimSpace(out))
+	}
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /signer.pass", network, kind)); err == nil {
+		keyPass = string(bytes.TrimSpace(out))
+	}
+	// Run a sanity check to see if the devp2p is reachable
+	port := infos.portmap[infos.envvars["FULL_PORT"]]
+	if err = checkPort(client.server, port); err != nil {
+		log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
+	}
+	// Assemble and return the useful infos
+	stats := &nodeInfos{
+		genesis:    genesis,
+		datadir:    infos.volumes["/root/.ethereum"],
+		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
+		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
+		peersTotal: totalPeers,
+		peersLight: lightPeers,
+		ethstats:   infos.envvars["STATS_NAME"],
+		etherbase:  infos.envvars["MINER_NAME"],
+		keyJSON:    keyJSON,
+		keyPass:    keyPass,
+		gasTarget:  gasTarget,
+		gasPrice:   gasPrice,
+	}
+	stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
+	if stats.portLight != 0 {
+		stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
+	}
+	return stats, nil
+}

cmd/puppeth/puppeth.go

@@ -0,0 +1,55 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// puppeth is a command to assemble and maintain private networks.
+package main
+
+import (
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/expanse-org/go-expanse/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// main is just a boring entry point to set up the CLI app.
+func main() {
+	app := cli.NewApp()
+	app.Name = "puppeth"
+	app.Usage = "assemble and maintain private Ethereum networks"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "network",
+			Usage: "name of the network to administer",
+		},
+		cli.IntFlag{
+			Name:  "loglevel",
+			Value: 4,
+			Usage: "log level to emit to the screen",
+		},
+	}
+	app.Action = func(c *cli.Context) error {
+		// Set up the logger to print everything and the random generator
+		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
+		rand.Seed(time.Now().UnixNano())
+
+		// Start the wizard and relinquish control
+		makeWizard(c.String("network")).run()
+		return nil
+	}
+	app.Run(os.Args)
+}

cmd/puppeth/ssh.go

@@ -0,0 +1,234 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/expanse-org/go-expanse/log"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// sshClient is a small wrapper around Go's SSH client with a few utility methods
+// implemented on top.
+type sshClient struct {
+	server  string // Server name or IP without port number
+	address string // IP address of the remote server
+	pubkey  []byte // RSA public key to authenticate the server
+	client  *ssh.Client
+	logger  log.Logger
+}
+
+// dial establishes an SSH connection to a remote node using the current user and
+// the user's configured private RSA key. If that fails, password authentication
+// is fallen back to. The caller may override the login user via user@server:port.
+func dial(server string, pubkey []byte) (*sshClient, error) {
+	// Figure out a label for the server and a logger
+	label := server
+	if strings.Contains(label, ":") {
+		label = label[:strings.Index(label, ":")]
+	}
+	login := ""
+	if strings.Contains(server, "@") {
+		login = label[:strings.Index(label, "@")]
+		label = label[strings.Index(label, "@")+1:]
+		server = server[strings.Index(server, "@")+1:]
+	}
+	logger := log.New("server", label)
+	logger.Debug("Attempting to establish SSH connection")
+
+	user, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+	if login == "" {
+		login = user.Username
+	}
+	// Configure the supported authentication methods (private key and password)
+	var auths []ssh.AuthMethod
+
+	path := filepath.Join(user.HomeDir, ".ssh", "id_rsa")
+	if buf, err := ioutil.ReadFile(path); err != nil {
+		log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
+	} else {
+		key, err := ssh.ParsePrivateKey(buf)
+		if err != nil {
+			log.Warn("Bad SSH key, falling back to passwords", "path", path, "err", err)
+		} else {
+			auths = append(auths, ssh.PublicKeys(key))
+		}
+	}
+	auths = append(auths, ssh.PasswordCallback(func() (string, error) {
+		fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server)
+		blob, err := terminal.ReadPassword(int(syscall.Stdin))
+
+		fmt.Println()
+		return string(blob), err
+	}))
+	// Resolve the IP address of the remote server
+	addr, err := net.LookupHost(label)
+	if err != nil {
+		return nil, err
+	}
+	if len(addr) == 0 {
+		return nil, errors.New("no IPs associated with domain")
+	}
+	// Try to dial in to the remote server
+	logger.Trace("Dialing remote SSH server", "user", login)
+	if !strings.Contains(server, ":") {
+		server += ":22"
+	}
+	keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		// If no public key is known for SSH, ask the user to confirm
+		if pubkey == nil {
+			fmt.Printf("The authenticity of host '%s (%s)' can't be established.\n", hostname, remote)
+			fmt.Printf("SSH key fingerprint is %s [MD5]\n", ssh.FingerprintLegacyMD5(key))
+			fmt.Printf("Are you sure you want to continue connecting (yes/no)? ")
+
+			text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+			switch {
+			case err != nil:
+				return err
+			case strings.TrimSpace(text) == "yes":
+				pubkey = key.Marshal()
+				return nil
+			default:
+				return fmt.Errorf("unknown auth choice: %v", text)
+			}
+		}
+		// If a public key exists for this SSH server, check that it matches
+		if bytes.Compare(pubkey, key.Marshal()) == 0 {
+			return nil
+		}
+		// We have a mismatch, forbid connecting
+		return errors.New("ssh key mismatch, readd the machine to update")
+	}
+	client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck})
+	if err != nil {
+		return nil, err
+	}
+	// Connection established, return our utility wrapper
+	c := &sshClient{
+		server:  label,
+		address: addr[0],
+		pubkey:  pubkey,
+		client:  client,
+		logger:  logger,
+	}
+	if err := c.init(); err != nil {
+		client.Close()
+		return nil, err
+	}
+	return c, nil
+}
+
+// init runs some initialization commands on the remote server to ensure it's
+// capable of acting as puppeth target.
+func (client *sshClient) init() error {
+	client.logger.Debug("Verifying if docker is available")
+	if out, err := client.Run("docker version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker configured incorrectly: %s", out)
+	}
+	client.logger.Debug("Verifying if docker-compose is available")
+	if out, err := client.Run("docker-compose version"); err != nil {
+		if len(out) == 0 {
+			return err
+		}
+		return fmt.Errorf("docker-compose configured incorrectly: %s", out)
+	}
+	return nil
+}
+
+// Close terminates the connection to an SSH server.
+func (client *sshClient) Close() error {
+	return client.client.Close()
+}
+
+// Run executes a command on the remote server and returns the combined output
+// along with any error status.
+func (client *sshClient) Run(cmd string) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Execute the command and return any output
+	client.logger.Trace("Running command on remote server", "cmd", cmd)
+	return session.CombinedOutput(cmd)
+}
+
+// Stream executes a command on the remote server and streams all outputs into
+// the local stdout and stderr streams.
+func (client *sshClient) Stream(cmd string) error {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	session.Stdout = os.Stdout
+	session.Stderr = os.Stderr
+
+	// Execute the command and return any output
+	client.logger.Trace("Streaming command on remote server", "cmd", cmd)
+	return session.Run(cmd)
+}
+
+// Upload copied the set of files to a remote server via SCP, creating any non-
+// existing folder in te mean time.
+func (client *sshClient) Upload(files map[string][]byte) ([]byte, error) {
+	// Establish a single command session
+	session, err := client.client.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	defer session.Close()
+
+	// Create a goroutine that streams the SCP content
+	go func() {
+		out, _ := session.StdinPipe()
+		defer out.Close()
+
+		for file, content := range files {
+			client.logger.Trace("Uploading file to server", "file", file, "bytes", len(content))
+
+			fmt.Fprintln(out, "D0755", 0, filepath.Dir(file))             // Ensure the folder exists
+			fmt.Fprintln(out, "C0644", len(content), filepath.Base(file)) // Create the actual file
+			out.Write(content)                                            // Stream the data content
+			fmt.Fprint(out, "\x00")                                       // Transfer end with \x00
+			fmt.Fprintln(out, "E")                                        // Leave directory (simpler)
+		}
+	}()
+	return session.CombinedOutput("/usr/bin/scp -v -tr ./")
+}

cmd/puppeth/wizard.go

@@ -0,0 +1,281 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// config contains all the configurations needed by puppeth that should be saved
+// between sessions.
+type config struct {
+	path      string        // File containing the configuration values
+	genesis   *core.Genesis // Genesis block to cache for node deploys
+	bootFull  []string      // Bootnodes to always connect to by full nodes
+	bootLight []string      // Bootnodes to always connect to by light nodes
+	ethstats  string        // Ethstats settings to cache for node deploys
+
+	Servers map[string][]byte `json:"servers,omitempty"`
+}
+
+// servers retrieves an alphabetically sorted list of servers.
+func (c config) servers() []string {
+	servers := make([]string, 0, len(c.Servers))
+	for server := range c.Servers {
+		servers = append(servers, server)
+	}
+	sort.Strings(servers)
+
+	return servers
+}
+
+// flush dumps the contents of config to disk.
+func (c config) flush() {
+	os.MkdirAll(filepath.Dir(c.path), 0755)
+
+	out, _ := json.MarshalIndent(c, "", "  ")
+	if err := ioutil.WriteFile(c.path, out, 0644); err != nil {
+		log.Warn("Failed to save puppeth configs", "file", c.path, "err", err)
+	}
+}
+
+type wizard struct {
+	network string // Network name to manage
+	conf    config // Configurations from previous runs
+
+	servers  map[string]*sshClient // SSH connections to servers to administer
+	services map[string][]string   // Ethereum services known to be running on servers
+
+	in *bufio.Reader // Wrapper around stdin to allow reading user input
+}
+
+// read reads a single line from stdin, trimming if from spaces.
+func (w *wizard) read() string {
+	fmt.Printf("> ")
+	text, err := w.in.ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+	return strings.TrimSpace(text)
+}
+
+// readString reads a single line from stdin, trimming if from spaces, enforcing
+// non-emptyness.
+func (w *wizard) readString() string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+	}
+}
+
+// readDefaultString reads a single line from stdin, trimming if from spaces. If
+// an empty line is entered, the default value is returned.
+func (w *wizard) readDefaultString(def string) string {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text != "" {
+			return text
+		}
+		return def
+	}
+}
+
+// readInt reads a single line from stdin, trimming if from spaces, enforcing it
+// to parse into an integer.
+func (w *wizard) readInt() int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			continue
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readDefaultInt reads a single line from stdin, trimming if from spaces, enforcing
+// it to parse into an integer. If an empty line is entered, the default value is
+// returned.
+func (w *wizard) readDefaultInt(def int) int {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, err := strconv.Atoi(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected integer", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readFloat reads a single line from stdin, trimming if from spaces, enforcing it
+// to parse into a float.
+func (w *wizard) readFloat() float64 {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			continue
+		}
+		val, err := strconv.ParseFloat(strings.TrimSpace(text), 64)
+		if err != nil {
+			log.Error("Invalid input, expected float", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readDefaultFloat reads a single line from stdin, trimming if from spaces, enforcing
+// it to parse into a float. If an empty line is entered, the default value is returned.
+func (w *wizard) readDefaultFloat(def float64) float64 {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		val, err := strconv.ParseFloat(strings.TrimSpace(text), 64)
+		if err != nil {
+			log.Error("Invalid input, expected float", "err", err)
+			continue
+		}
+		return val
+	}
+}
+
+// readPassword reads a single line from stdin, trimming it from the trailing new
+// line and returns it. The input will not be echoed.
+func (w *wizard) readPassword() string {
+	for {
+		fmt.Printf("> ")
+		text, err := terminal.ReadPassword(int(syscall.Stdin))
+		if err != nil {
+			log.Crit("Failed to read password", "err", err)
+		}
+		fmt.Println()
+		return string(text)
+	}
+}
+
+// readAddress reads a single line from stdin, trimming if from spaces and converts
+// it to an Ethereum address.
+func (w *wizard) readAddress() *common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return nil
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		address := common.BigToAddress(bigaddr)
+		return &address
+	}
+}
+
+// readDefaultAddress reads a single line from stdin, trimming if from spaces and
+// converts it to an Ethereum address. If an empty line is entered, the default
+// value is returned.
+func (w *wizard) readDefaultAddress(def common.Address) common.Address {
+	for {
+		// Read the address from the user
+		fmt.Printf("> 0x")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.TrimSpace(text); text == "" {
+			return def
+		}
+		// Make sure it looks ok and return it if so
+		if len(text) != 40 {
+			log.Error("Invalid address length, please retry")
+			continue
+		}
+		bigaddr, _ := new(big.Int).SetString(text, 16)
+		return common.BigToAddress(bigaddr)
+	}
+}
+
+// readJSON reads a raw JSON message and returns it.
+func (w *wizard) readJSON() string {
+	var blob json.RawMessage
+
+	for {
+		fmt.Printf("> ")
+		if err := json.NewDecoder(w.in).Decode(&blob); err != nil {
+			log.Error("Invalid JSON, please try again", "err", err)
+			continue
+		}
+		return string(blob)
+	}
+}

cmd/puppeth/wizard_dashboard.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployDashboard queries the user for various input on deploying a web-service
+// dashboard, after which is pushes the container.
+func (w *wizard) deployDashboard() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active dashboard configurations from the server
+	infos, err := checkDashboard(client, w.network)
+	if err != nil {
+		infos = &dashboardInfos{
+			port: 80,
+			host: client.server,
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the dashboard listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy the dashboard on
+	infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host)
+	if err != nil {
+		log.Error("Failed to decide on dashboard host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out which services are available
+	available := make(map[string][]string)
+	for server, services := range w.services {
+		for _, service := range services {
+			available[service] = append(available[service], server)
+		}
+	}
+	listing := make(map[string]string)
+	for _, service := range []string{"ethstats", "explorer", "wallet", "faucet"} {
+		// Gather all the locally hosted pages of this type
+		var pages []string
+		for _, server := range available[service] {
+			client := w.servers[server]
+			if client == nil {
+				continue
+			}
+			// If there's a service running on the machine, retrieve it's port number
+			var port int
+			switch service {
+			case "ethstats":
+				if infos, err := checkEthstats(client, w.network); err == nil {
+					port = infos.port
+				}
+			case "faucet":
+				if infos, err := checkFaucet(client, w.network); err == nil {
+					port = infos.port
+				}
+			}
+			if page, err := resolve(client, w.network, service, port); err == nil && page != "" {
+				pages = append(pages, page)
+			}
+		}
+		// Promt the user to chose one, enter manually or simply not list this service
+		defLabel, defChoice := "don't list", len(pages)+2
+		if len(pages) > 0 {
+			defLabel, defChoice = pages[0], 1
+		}
+		fmt.Println()
+		fmt.Printf("Which %s service to list? (default = %s)\n", service, defLabel)
+		for i, page := range pages {
+			fmt.Printf(" %d. %s\n", i+1, page)
+		}
+		fmt.Printf(" %d. List external %s service\n", len(pages)+1, service)
+		fmt.Printf(" %d. Don't list any %s service\n", len(pages)+2, service)
+
+		choice := w.readDefaultInt(defChoice)
+		if choice < 0 || choice > len(pages)+2 {
+			log.Error("Invalid listing choice, aborting")
+			return
+		}
+		switch {
+		case choice <= len(pages):
+			listing[service] = pages[choice-1]
+		case choice == len(pages)+1:
+			fmt.Println()
+			fmt.Printf("Which address is the external %s service at?\n", service)
+			listing[service] = w.readString()
+		default:
+			// No service hosting for this
+		}
+	}
+	// If we have ethstats running, ask whether to make the secret public or not
+	var ethstats bool
+	if w.conf.ethstats != "" {
+		fmt.Println()
+		fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)")
+		ethstats = w.readDefaultString("y") == "y"
+	}
+	// Try to deploy the dashboard container on the host
+	if out, err := deployDashboard(client, w.network, infos.port, infos.host, listing, &w.conf, ethstats); err != nil {
+		log.Error("Failed to deploy dashboard container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_ethstats.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployEthstats queries the user for various input on deploying an ethstats
+// monitoring server, after which it executes it.
+func (w *wizard) deployEthstats() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkEthstats(client, w.network)
+	if err != nil {
+		infos = &ethstatsInfos{
+			port:   80,
+			host:   client.server,
+			secret: "",
+		}
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should ethstats listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on ethstats host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the secret and boot ethstats
+	fmt.Println()
+	if infos.secret == "" {
+		fmt.Printf("What should be the secret password for the API? (must not be empty)\n")
+		infos.secret = w.readString()
+	} else {
+		fmt.Printf("What should be the secret password for the API? (default = %s)\n", infos.secret)
+		infos.secret = w.readDefaultString(infos.secret)
+	}
+	// Try to deploy the ethstats server on the host
+	trusted := make([]string, 0, len(w.servers))
+	for _, client := range w.servers {
+		if client != nil {
+			trusted = append(trusted, client.address)
+		}
+	}
+	if out, err := deployEthstats(client, w.network, infos.port, infos.secret, infos.host, trusted); err != nil {
+		log.Error("Failed to deploy ethstats container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_faucet.go

@@ -0,0 +1,203 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployFaucet queries the user for various input on deploying a faucet, after
+// which it executes it.
+func (w *wizard) deployFaucet() {
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active faucet configurations from the server
+	infos, err := checkFaucet(client, w.network)
+	if err != nil {
+		infos = &faucetInfos{
+			node:    &nodeInfos{portFull: 30303, peersTotal: 25},
+			port:    80,
+			host:    client.server,
+			amount:  1,
+			minutes: 1440,
+			tiers:   3,
+		}
+	}
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.node.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the faucet listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
+		log.Error("Failed to decide on faucet host", "err", err)
+		return
+	}
+	// Port and proxy settings retrieved, figure out the funcing amount per perdion configurations
+	fmt.Println()
+	fmt.Printf("How many Ethers to release per request? (default = %d)\n", infos.amount)
+	infos.amount = w.readDefaultInt(infos.amount)
+
+	fmt.Println()
+	fmt.Printf("How many minutes to enforce between requests? (default = %d)\n", infos.minutes)
+	infos.minutes = w.readDefaultInt(infos.minutes)
+
+	fmt.Println()
+	fmt.Printf("How many funding tiers to feature (x2.5 amounts, x3 timeout)? (default = %d)\n", infos.tiers)
+	infos.tiers = w.readDefaultInt(infos.tiers)
+	if infos.tiers == 0 {
+		log.Error("At least one funding tier must be set")
+		return
+	}
+	// Accessing GitHub gists requires API authorization, retrieve it
+	if infos.githubUser != "" {
+		fmt.Println()
+		fmt.Printf("Reuse previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser)
+		if w.readDefaultString("y") != "y" {
+			infos.githubUser, infos.githubToken = "", ""
+		}
+	}
+	if infos.githubUser == "" {
+		// No previous authorization (or new one requested)
+		fmt.Println()
+		fmt.Println("Which GitHub user to verify Gists through?")
+		infos.githubUser = w.readString()
+
+		fmt.Println()
+		fmt.Println("What is the GitHub personal access token of the user? (won't be echoed)")
+		infos.githubToken = w.readPassword()
+
+		// Do a sanity check query against github to ensure it's valid
+		req, _ := http.NewRequest("GET", "https://api.github.com/user", nil)
+		req.SetBasicAuth(infos.githubUser, infos.githubToken)
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Error("Failed to verify GitHub authentication", "err", err)
+			return
+		}
+		defer res.Body.Close()
+
+		var msg struct {
+			Login   string `json:"login"`
+			Message string `json:"message"`
+		}
+		if err = json.NewDecoder(res.Body).Decode(&msg); err != nil {
+			log.Error("Failed to decode authorization response", "err", err)
+			return
+		}
+		if msg.Login != infos.githubUser {
+			log.Error("GitHub authorization failed", "user", infos.githubUser, "message", msg.Message)
+			return
+		}
+	}
+	// Accessing the reCaptcha service requires API authorizations, request it
+	if infos.captchaToken != "" {
+		fmt.Println()
+		fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)")
+		if w.readDefaultString("y") != "y" {
+			infos.captchaToken, infos.captchaSecret = "", ""
+		}
+	}
+	if infos.captchaToken == "" {
+		// No previous authorization (or old one discarded)
+		fmt.Println()
+		fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)")
+		if w.readDefaultString("n") == "y" {
+			// Captcha protection explicitly requested, read the site and secret keys
+			fmt.Println()
+			fmt.Printf("What is the reCaptcha site key to authenticate human users?\n")
+			infos.captchaToken = w.readString()
+
+			fmt.Println()
+			fmt.Printf("What is the reCaptcha secret key to verify authentications? (won't be echoed)\n")
+			infos.captchaSecret = w.readPassword()
+		}
+	}
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.node.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.node.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.node.datadir)
+		infos.node.datadir = w.readDefaultString(infos.node.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
+	infos.node.portFull = w.readDefaultInt(infos.node.portFull)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.node.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.node.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.node.ethstats)
+		infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats
+	}
+	// Load up the credential needed to release funds
+	if infos.node.keyJSON != "" {
+		var key keystore.Key
+		if err := json.Unmarshal([]byte(infos.node.keyJSON), &key); err != nil {
+			infos.node.keyJSON, infos.node.keyPass = "", ""
+		} else {
+			fmt.Println()
+			fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex())
+			if w.readDefaultString("y") != "y" {
+				infos.node.keyJSON, infos.node.keyPass = "", ""
+			}
+		}
+	}
+	if infos.node.keyJSON == "" {
+		fmt.Println()
+		fmt.Println("Please paste the faucet's funding account key JSON:")
+		infos.node.keyJSON = w.readJSON()
+
+		fmt.Println()
+		fmt.Println("What's the unlock password for the account? (won't be echoed)")
+		infos.node.keyPass = w.readPassword()
+
+		if _, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil {
+			log.Error("Failed to decrypt key with given passphrase")
+			return
+		}
+	}
+	// Try to deploy the faucet server on the host
+	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos); err != nil {
+		log.Error("Failed to deploy faucet container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	w.networkStats(false)
+}

cmd/puppeth/wizard_genesis.go

@@ -0,0 +1,136 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"time"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+// makeGenesis creates a new genesis struct based on some user input.
+func (w *wizard) makeGenesis() {
+	// Construct a default genesis block
+	genesis := &core.Genesis{
+		Timestamp:  uint64(time.Now().Unix()),
+		GasLimit:   4700000,
+		Difficulty: big.NewInt(1048576),
+		Alloc:      make(core.GenesisAlloc),
+		Config: &params.ChainConfig{
+			HomesteadBlock: big.NewInt(1),
+			EIP150Block:    big.NewInt(2),
+			EIP155Block:    big.NewInt(3),
+			EIP158Block:    big.NewInt(3),
+		},
+	}
+	// Figure out which consensus engine to choose
+	fmt.Println()
+	fmt.Println("Which consensus engine to use? (default = clique)")
+	fmt.Println(" 1. Ethash - proof-of-work")
+	fmt.Println(" 2. Clique - proof-of-authority")
+
+	choice := w.read()
+	switch {
+	case choice == "1":
+		// In case of ethash, we're pretty much done
+		genesis.Config.Ethash = new(params.EthashConfig)
+		genesis.ExtraData = make([]byte, 32)
+
+	case choice == "" || choice == "2":
+		// In the case of clique, configure the consensus parameters
+		genesis.Difficulty = big.NewInt(1)
+		genesis.Config.Clique = &params.CliqueConfig{
+			Period: 15,
+			Epoch:  30000,
+		}
+		fmt.Println()
+		fmt.Println("How many seconds should blocks take? (default = 15)")
+		genesis.Config.Clique.Period = uint64(w.readDefaultInt(15))
+
+		// We also need the initial list of signers
+		fmt.Println()
+		fmt.Println("Which accounts are allowed to seal? (mandatory at least one)")
+
+		var signers []common.Address
+		for {
+			if address := w.readAddress(); address != nil {
+				signers = append(signers, *address)
+				continue
+			}
+			if len(signers) > 0 {
+				break
+			}
+		}
+		// Sort the signers and embed into the extra-data section
+		for i := 0; i < len(signers); i++ {
+			for j := i + 1; j < len(signers); j++ {
+				if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
+					signers[i], signers[j] = signers[j], signers[i]
+				}
+			}
+		}
+		genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65)
+		for i, signer := range signers {
+			copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:])
+		}
+
+	default:
+		log.Crit("Invalid consensus engine choice", "choice", choice)
+	}
+	// Consensus all set, just ask for initial funds and go
+	fmt.Println()
+	fmt.Println("Which accounts should be pre-funded? (advisable at least one)")
+	for {
+		// Read the address of the account to fund
+		if address := w.readAddress(); address != nil {
+			genesis.Alloc[*address] = core.GenesisAccount{
+				Balance: new(big.Int).Lsh(big.NewInt(1), 256-7), // 2^256 / 128 (allow many pre-funds without balance overflows)
+			}
+			continue
+		}
+		break
+	}
+	// Add a batch of precompile balances to avoid them getting deleted
+	for i := int64(0); i < 256; i++ {
+		genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
+	}
+	fmt.Println()
+
+	// Query the user for some custom extras
+	fmt.Println()
+	fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)")
+	genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536))))
+
+	fmt.Println()
+	fmt.Println("Anything fun to embed into the genesis block? (max 32 bytes)")
+
+	extra := w.read()
+	if len(extra) > 32 {
+		extra = extra[:32]
+	}
+	genesis.ExtraData = append([]byte(extra), genesis.ExtraData[len(extra):]...)
+
+	// All done, store the genesis and flush to disk
+	w.conf.genesis = genesis
+}

cmd/puppeth/wizard_intro.go

@@ -0,0 +1,156 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// makeWizard creates and returns a new puppeth wizard.
+func makeWizard(network string) *wizard {
+	return &wizard{
+		network: network,
+		conf: config{
+			Servers: make(map[string][]byte),
+		},
+		servers:  make(map[string]*sshClient),
+		services: make(map[string][]string),
+		in:       bufio.NewReader(os.Stdin),
+	}
+}
+
+// run displays some useful infos to the user, starting on the journey of
+// setting up a new or managing an existing Ethereum private network.
+func (w *wizard) run() {
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println("| Welcome to puppeth, your Ethereum private network manager |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| This tool lets you create a new Ethereum network down to  |")
+	fmt.Println("| the genesis block, bootnodes, miners and ethstats servers |")
+	fmt.Println("| without the hassle that it would normally entail.         |")
+	fmt.Println("|                                                           |")
+	fmt.Println("| Puppeth uses SSH to dial in to remote servers, and builds |")
+	fmt.Println("| its network components out of Docker containers using the |")
+	fmt.Println("| docker-compose toolset.                                   |")
+	fmt.Println("+-----------------------------------------------------------+")
+	fmt.Println()
+
+	// Make sure we have a good network name to work with	fmt.Println()
+	if w.network == "" {
+		fmt.Println("Please specify a network name to administer (no spaces, please)")
+		for {
+			w.network = w.readString()
+			if !strings.Contains(w.network, " ") {
+				fmt.Printf("Sweet, you can set this via --network=%s next time!\n\n", w.network)
+				break
+			}
+			log.Error("I also like to live dangerously, still no spaces")
+		}
+	}
+	log.Info("Administering Ethereum network", "name", w.network)
+
+	// Load initial configurations and connect to all live servers
+	w.conf.path = filepath.Join(os.Getenv("HOME"), ".puppeth", w.network)
+
+	blob, err := ioutil.ReadFile(w.conf.path)
+	if err != nil {
+		log.Warn("No previous configurations found", "path", w.conf.path)
+	} else if err := json.Unmarshal(blob, &w.conf); err != nil {
+		log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err)
+	} else {
+		for server, pubkey := range w.conf.Servers {
+			log.Info("Dialing previously configured server", "server", server)
+			client, err := dial(server, pubkey)
+			if err != nil {
+				log.Error("Previous server unreachable", "server", server, "err", err)
+			}
+			w.servers[server] = client
+		}
+		w.networkStats(false)
+	}
+	// Basics done, loop ad infinitum about what to do
+	for {
+		fmt.Println()
+		fmt.Println("What would you like to do? (default = stats)")
+		fmt.Println(" 1. Show network stats")
+		if w.conf.genesis == nil {
+			fmt.Println(" 2. Configure new genesis")
+		} else {
+			fmt.Println(" 2. Save existing genesis")
+		}
+		if len(w.servers) == 0 {
+			fmt.Println(" 3. Track new remote server")
+		} else {
+			fmt.Println(" 3. Manage tracked machines")
+		}
+		if len(w.services) == 0 {
+			fmt.Println(" 4. Deploy network components")
+		} else {
+			fmt.Println(" 4. Manage network components")
+		}
+		//fmt.Println(" 5. ProTips for common usecases")
+
+		choice := w.read()
+		switch {
+		case choice == "" || choice == "1":
+			w.networkStats(false)
+
+		case choice == "2":
+			// If we don't have a genesis, make one
+			if w.conf.genesis == nil {
+				w.makeGenesis()
+			} else {
+				// Otherwise just save whatever we currently have
+				fmt.Println()
+				fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
+				out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
+				if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
+					log.Error("Failed to save genesis file", "err", err)
+				}
+				log.Info("Exported existing genesis block")
+			}
+		case choice == "3":
+			if len(w.servers) == 0 {
+				if w.makeServer() != "" {
+					w.networkStats(false)
+				}
+			} else {
+				w.manageServers()
+			}
+		case choice == "4":
+			if len(w.services) == 0 {
+				w.deployComponent()
+			} else {
+				w.manageComponents()
+			}
+
+		case choice == "5":
+			w.networkStats(true)
+
+		default:
+			log.Error("That's not something I can do")
+		}
+	}
+}

cmd/puppeth/wizard_netstats.go

@@ -0,0 +1,235 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/olekukonko/tablewriter"
+)
+
+// networkStats verifies the status of network components and generates a protip
+// configuration set to give users hints on how to do various tasks.
+func (w *wizard) networkStats(tips bool) {
+	if len(w.servers) == 0 {
+		log.Error("No remote machines to gather stats from")
+		return
+	}
+	protips := new(protips)
+
+	// Iterate over all the specified hosts and check their status
+	stats := tablewriter.NewWriter(os.Stdout)
+	stats.SetHeader([]string{"Server", "IP", "Status", "Service", "Details"})
+	stats.SetColWidth(100)
+
+	for server, pubkey := range w.conf.Servers {
+		client := w.servers[server]
+		logger := log.New("server", server)
+		logger.Info("Starting remote server health-check")
+
+		// If the server is not connected, try to connect again
+		if client == nil {
+			conn, err := dial(server, pubkey)
+			if err != nil {
+				logger.Error("Failed to establish remote connection", "err", err)
+				stats.Append([]string{server, "", err.Error(), "", ""})
+				continue
+			}
+			client = conn
+		}
+		// Client connected one way or another, run health-checks
+		services := make(map[string]string)
+		logger.Debug("Checking for nginx availability")
+		if infos, err := checkNginx(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["nginx"] = err.Error()
+			}
+		} else {
+			services["nginx"] = infos.String()
+		}
+		logger.Debug("Checking for ethstats availability")
+		if infos, err := checkEthstats(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["ethstats"] = err.Error()
+			}
+		} else {
+			services["ethstats"] = infos.String()
+			protips.ethstats = infos.config
+		}
+		logger.Debug("Checking for bootnode availability")
+		if infos, err := checkNode(client, w.network, true); err != nil {
+			if err != ErrServiceUnknown {
+				services["bootnode"] = err.Error()
+			}
+		} else {
+			services["bootnode"] = infos.String()
+
+			protips.genesis = string(infos.genesis)
+			protips.bootFull = append(protips.bootFull, infos.enodeFull)
+			if infos.enodeLight != "" {
+				protips.bootLight = append(protips.bootLight, infos.enodeLight)
+			}
+		}
+		logger.Debug("Checking for sealnode availability")
+		if infos, err := checkNode(client, w.network, false); err != nil {
+			if err != ErrServiceUnknown {
+				services["sealnode"] = err.Error()
+			}
+		} else {
+			services["sealnode"] = infos.String()
+			protips.genesis = string(infos.genesis)
+		}
+		logger.Debug("Checking for faucet availability")
+		if infos, err := checkFaucet(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["faucet"] = err.Error()
+			}
+		} else {
+			services["faucet"] = infos.String()
+		}
+		logger.Debug("Checking for dashboard availability")
+		if infos, err := checkDashboard(client, w.network); err != nil {
+			if err != ErrServiceUnknown {
+				services["dashboard"] = err.Error()
+			}
+		} else {
+			services["dashboard"] = infos.String()
+		}
+		// All status checks complete, report and check next server
+		delete(w.services, server)
+		for service := range services {
+			w.services[server] = append(w.services[server], service)
+		}
+		server, address := client.server, client.address
+		for service, status := range services {
+			stats.Append([]string{server, address, "online", service, status})
+			server, address = "", ""
+		}
+		if len(services) == 0 {
+			stats.Append([]string{server, address, "online", "", ""})
+		}
+	}
+	// If a genesis block was found, load it into our configs
+	if protips.genesis != "" {
+		genesis := new(core.Genesis)
+		if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
+			log.Error("Failed to parse remote genesis", "err", err)
+		} else {
+			w.conf.genesis = genesis
+			protips.network = genesis.Config.ChainId.Int64()
+		}
+	}
+	if protips.ethstats != "" {
+		w.conf.ethstats = protips.ethstats
+	}
+	w.conf.bootFull = protips.bootFull
+	w.conf.bootLight = protips.bootLight
+
+	// Print any collected stats and return
+	if !tips {
+		stats.Render()
+	} else {
+		protips.print(w.network)
+	}
+}
+
+// protips contains a collection of network infos to report pro-tips
+// based on.
+type protips struct {
+	genesis   string
+	network   int64
+	bootFull  []string
+	bootLight []string
+	ethstats  string
+}
+
+// print analyzes the network information available and prints a collection of
+// pro tips for the user's consideration.
+func (p *protips) print(network string) {
+	// If a known genesis block is available, display it and prepend an init command
+	fullinit, lightinit := "", ""
+	if p.genesis != "" {
+		fullinit = fmt.Sprintf("geth --datadir=$HOME/.%s init %s.json && ", network, network)
+		lightinit = fmt.Sprintf("geth --datadir=$HOME/.%s --light init %s.json && ", network, network)
+	}
+	// If an ethstats server is available, add the ethstats flag
+	statsflag := ""
+	if p.ethstats != "" {
+		if strings.Contains(p.ethstats, " ") {
+			statsflag = fmt.Sprintf(` --ethstats="yournode:%s"`, p.ethstats)
+		} else {
+			statsflag = fmt.Sprintf(` --ethstats=yournode:%s`, p.ethstats)
+		}
+	}
+	// If bootnodes have been specified, add the bootnode flag
+	bootflagFull := ""
+	if len(p.bootFull) > 0 {
+		bootflagFull = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootFull, ","))
+	}
+	bootflagLight := ""
+	if len(p.bootLight) > 0 {
+		bootflagLight = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootLight, ","))
+	}
+	// Assemble all the known pro-tips
+	var tasks, tips []string
+
+	tasks = append(tasks, "Run an archive node with historical data")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=1024%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a full node with recent data only")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=512 --fast%s%s", fullinit, p.network, network, statsflag, bootflagFull))
+
+	tasks = append(tasks, "Run a light node with on demand retrievals")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	tasks = append(tasks, "Run an embedded node with constrained memory")
+	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=32 --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
+
+	// If the tips are short, display in a table
+	short := true
+	for _, tip := range tips {
+		if len(tip) > 100 {
+			short = false
+			break
+		}
+	}
+	fmt.Println()
+	if short {
+		howto := tablewriter.NewWriter(os.Stdout)
+		howto.SetHeader([]string{"Fun tasks for you", "Tips on how to"})
+		howto.SetColWidth(100)
+
+		for i := 0; i < len(tasks); i++ {
+			howto.Append([]string{tasks[i], tips[i]})
+		}
+		howto.Render()
+		return
+	}
+	// Meh, tips got ugly, split into many lines
+	for i := 0; i < len(tasks); i++ {
+		fmt.Println(tasks[i])
+		fmt.Println(strings.Repeat("-", len(tasks[i])))
+		fmt.Println(tips[i])
+		fmt.Println()
+		fmt.Println()
+	}
+}

cmd/puppeth/wizard_network.go

@@ -0,0 +1,198 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// manageServers displays a list of servers the user can disconnect from, and an
+// option to connect to new servers.
+func (w *wizard) manageServers() {
+	// List all the servers we can disconnect, along with an entry to connect a new one
+	fmt.Println()
+
+	servers := w.conf.servers()
+	for i, server := range servers {
+		fmt.Printf(" %d. Disconnect %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return
+	}
+	// If the user selected an existing server, drop it
+	if choice <= len(w.conf.Servers) {
+		server := servers[choice-1]
+		client := w.servers[server]
+
+		delete(w.servers, server)
+		if client != nil {
+			client.Close()
+		}
+		delete(w.conf.Servers, server)
+		w.conf.flush()
+
+		log.Info("Disconnected existing server", "server", server)
+		w.networkStats(false)
+		return
+	}
+	// If the user requested connecting a new server, do it
+	if w.makeServer() != "" {
+		w.networkStats(false)
+	}
+}
+
+// makeServer reads a single line from stdin and interprets it as a hostname to
+// connect to. It tries to establish a new SSH session and also executing some
+// baseline validations.
+//
+// If connection succeeds, the server is added to the wizards configs!
+func (w *wizard) makeServer() string {
+	fmt.Println()
+	fmt.Println("Please enter remote server's address:")
+
+	for {
+		// Read and fial the server to ensure docker is present
+		input := w.readString()
+
+		client, err := dial(input, nil)
+		if err != nil {
+			log.Error("Server not ready for puppeth", "err", err)
+			return ""
+		}
+		// All checks passed, start tracking the server
+		w.servers[input] = client
+		w.conf.Servers[input] = client.pubkey
+		w.conf.flush()
+
+		return input
+	}
+}
+
+// selectServer lists the user all the currnetly known servers to choose from,
+// also granting the option to add a new one.
+func (w *wizard) selectServer() string {
+	// List the available server to the user and wait for a choice
+	fmt.Println()
+	fmt.Println("Which server do you want to interact with?")
+
+	servers := w.conf.servers()
+	for i, server := range servers {
+		fmt.Printf(" %d. %s\n", i+1, server)
+	}
+	fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(w.conf.Servers)+1 {
+		log.Error("Invalid server choice, aborting")
+		return ""
+	}
+	// If the user requested connecting to a new server, go for it
+	if choice <= len(w.conf.Servers) {
+		return servers[choice-1]
+	}
+	return w.makeServer()
+}
+
+// manageComponents displays a list of network components the user can tear down
+// and an option
+func (w *wizard) manageComponents() {
+	// List all the componens we can tear down, along with an entry to deploy a new one
+	fmt.Println()
+
+	var serviceHosts, serviceNames []string
+	for server, services := range w.services {
+		for _, service := range services {
+			serviceHosts = append(serviceHosts, server)
+			serviceNames = append(serviceNames, service)
+
+			fmt.Printf(" %d. Tear down %s on %s\n", len(serviceHosts), strings.Title(service), server)
+		}
+	}
+	fmt.Printf(" %d. Deploy new network component\n", len(serviceHosts)+1)
+
+	choice := w.readInt()
+	if choice < 0 || choice > len(serviceHosts)+1 {
+		log.Error("Invalid component choice, aborting")
+		return
+	}
+	// If the user selected an existing service, destroy it
+	if choice <= len(serviceHosts) {
+		// Figure out the service to destroy and execute it
+		service := serviceNames[choice-1]
+		server := serviceHosts[choice-1]
+		client := w.servers[server]
+
+		if out, err := tearDown(client, w.network, service, true); err != nil {
+			log.Error("Failed to tear down component", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return
+		}
+		// Clean up any references to it from out state
+		services := w.services[server]
+		for i, name := range services {
+			if name == service {
+				w.services[server] = append(services[:i], services[i+1:]...)
+				if len(w.services[server]) == 0 {
+					delete(w.services, server)
+				}
+			}
+		}
+		log.Info("Torn down existing component", "server", server, "service", service)
+		return
+	}
+	// If the user requested deploying a new component, do it
+	w.deployComponent()
+}
+
+// deployComponent displays a list of network components the user can deploy and
+// guides through the process.
+func (w *wizard) deployComponent() {
+	// Print all the things we can deploy and wait or user choice
+	fmt.Println()
+	fmt.Println("What would you like to deploy? (recommended order)")
+	fmt.Println(" 1. Ethstats  - Network monitoring tool")
+	fmt.Println(" 2. Bootnode  - Entry point of the network")
+	fmt.Println(" 3. Sealer    - Full node minting new blocks")
+	fmt.Println(" 4. Wallet    - Browser wallet for quick sends (todo)")
+	fmt.Println(" 5. Faucet    - Crypto faucet to give away funds")
+	fmt.Println(" 6. Dashboard - Website listing above web-services")
+
+	switch w.read() {
+	case "1":
+		w.deployEthstats()
+	case "2":
+		w.deployNode(true)
+	case "3":
+		w.deployNode(false)
+	case "4":
+	case "5":
+		w.deployFaucet()
+	case "6":
+		w.deployDashboard()
+	default:
+		log.Error("That's not something I can do")
+	}
+}

cmd/puppeth/wizard_nginx.go

@@ -0,0 +1,58 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// ensureVirtualHost checks whether a reverse-proxy is running on the specified
+// host machine, and if yes requests a virtual host from the user to host a
+// specific web service on. If no proxy exists, the method will offer to deploy
+// one.
+//
+// If the user elects not to use a reverse proxy, an empty hostname is returned!
+func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (string, error) {
+	if proxy, _ := checkNginx(client, w.network); proxy != nil {
+		// Reverse proxy is running, if ports match, we need a virtual host
+		if proxy.port == port {
+			fmt.Println()
+			fmt.Printf("Shared port, which domain to assign? (default = %s)\n", def)
+			return w.readDefaultString(def), nil
+		}
+	}
+	// Reverse proxy is not running, offer to deploy a new one
+	fmt.Println()
+	fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)")
+	if w.readDefaultString("y") == "y" {
+		if out, err := deployNginx(client, w.network, port); err != nil {
+			log.Error("Failed to deploy reverse-proxy", "err", err)
+			if len(out) > 0 {
+				fmt.Printf("%s\n", out)
+			}
+			return "", err
+		}
+		// Reverse proxy deployed, ask again for the virtual-host
+		fmt.Println()
+		fmt.Printf("Proxy deployed, which domain to assign? (default = %s)\n", def)
+		return w.readDefaultString(def), nil
+	}
+	// Reverse proxy not requested, deploy as a standalone service
+	return "", nil
+}

cmd/puppeth/wizard_node.go

@@ -0,0 +1,160 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts/keystore"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// deployNode creates a new node configuration based on some user input.
+func (w *wizard) deployNode(boot bool) {
+	// Do some sanity check before the user wastes time on input
+	if w.conf.genesis == nil {
+		log.Error("No genesis block configured")
+		return
+	}
+	if w.conf.ethstats == "" {
+		log.Error("No ethstats server configured")
+		return
+	}
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active ethstats configurations from the server
+	infos, err := checkNode(client, w.network, boot)
+	if err != nil {
+		if boot {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
+		} else {
+			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
+		}
+	}
+	infos.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	infos.network = w.conf.genesis.Config.ChainId.Int64()
+
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		infos.datadir = w.readDefaultString(infos.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
+	infos.portFull = w.readDefaultInt(infos.portFull)
+
+	// Figure out how many peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many peers to allow connecting? (default = %d)\n", infos.peersTotal)
+	infos.peersTotal = w.readDefaultInt(infos.peersTotal)
+
+	// Figure out how many light peers to allow (different based on node type)
+	fmt.Println()
+	fmt.Printf("How many light peers to allow connecting? (default = %d)\n", infos.peersLight)
+	infos.peersLight = w.readDefaultInt(infos.peersLight)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.ethstats == "" {
+		fmt.Printf("What should the node be called on the stats page?\n")
+		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the node be called on the stats page? (default = %s)\n", infos.ethstats)
+		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+	}
+	// If the node is a miner/signer, load up needed credentials
+	if !boot {
+		if w.conf.genesis.Config.Ethash != nil {
+			// Ethash based miners only need an etherbase to mine against
+			fmt.Println()
+			if infos.etherbase == "" {
+				fmt.Printf("What address should the miner user?\n")
+				for {
+					if address := w.readAddress(); address != nil {
+						infos.etherbase = address.Hex()
+						break
+					}
+				}
+			} else {
+				fmt.Printf("What address should the miner user? (default = %s)\n", infos.etherbase)
+				infos.etherbase = w.readDefaultAddress(common.HexToAddress(infos.etherbase)).Hex()
+			}
+		} else if w.conf.genesis.Config.Clique != nil {
+			// If a previous signer was already set, offer to reuse it
+			if infos.keyJSON != "" {
+				if key, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
+					infos.keyJSON, infos.keyPass = "", ""
+				} else {
+					fmt.Println()
+					fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex())
+					if w.readDefaultString("y") != "y" {
+						infos.keyJSON, infos.keyPass = "", ""
+					}
+				}
+			}
+			// Clique based signers need a keyfile and unlock password, ask if unavailable
+			if infos.keyJSON == "" {
+				fmt.Println()
+				fmt.Println("Please paste the signer's key JSON:")
+				infos.keyJSON = w.readJSON()
+
+				fmt.Println()
+				fmt.Println("What's the unlock password for the account? (won't be echoed)")
+				infos.keyPass = w.readPassword()
+
+				if _, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
+					log.Error("Failed to decrypt key with given passphrase")
+					return
+				}
+			}
+		}
+		// Establish the gas dynamics to be enforced by the signer
+		fmt.Println()
+		fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget)
+		infos.gasTarget = w.readDefaultFloat(infos.gasTarget)
+
+		fmt.Println()
+		fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice)
+		infos.gasPrice = w.readDefaultFloat(infos.gasPrice)
+	}
+	// Try to deploy the full node on the host
+	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos); err != nil {
+		log.Error("Failed to deploy Ethereum node container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	log.Info("Waiting for node to finish booting")
+	time.Sleep(3 * time.Second)
+
+	w.networkStats(false)
+}

cmd/rlpdump/main.go

@@ -32,6 +32,7 @@ import (
 var (
 	hexMode = flag.String("hex", "", "dump given hex data")
 	noASCII = flag.Bool("noascii", false, "don't print ASCII strings readably")
+	single  = flag.Bool("single", false, "print only the first element, discard the rest")
 )
 
 func init() {
@@ -82,6 +83,9 @@ func main() {
 			break
 		}
 		fmt.Println()
+		if *single {
+			break
+		}
 	}
 }
 

cmd/swarm/cleandb.go

@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
+// This file is part of go-ethereum.
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
+// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
+// go-ethereum is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Lesser General Public License
+// You should have received a copy of the GNU General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/swarm/list.go

@@ -0,0 +1,61 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/expanse-org/go-expanse/cmd/utils"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	list, err := client.List(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}

cmd/swarm/main.go

@@ -39,19 +39,16 @@ import (
 	"github.com/expanse-org/go-expanse/node"
 	"github.com/expanse-org/go-expanse/p2p"
 	"github.com/expanse-org/go-expanse/p2p/discover"
+	"github.com/expanse-org/go-expanse/params"
 	"github.com/expanse-org/go-expanse/swarm"
 	bzzapi "github.com/expanse-org/go-expanse/swarm/api"
 	"gopkg.in/urfave/cli.v1"
 )
 
-const (
+const clientIdentifier = "swarm"
-	clientIdentifier = "swarm"
-	versionString    = "0.2"
-)
 
 var (
 	gitCommit        string // Git SHA1 commit hash of the release (set via linker flags)
-	app              = utils.NewApp(gitCommit, "Expanse Swarm")
 	testbetBootNodes = []string{
 		"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
 		"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
@@ -70,6 +67,10 @@ var (
 		Name:  "bzzaccount",
 		Usage: "Swarm account key file",
 	}
+	SwarmListenAddrFlag = cli.StringFlag{
+		Name:  "httpaddr",
+		Usage: "Swarm HTTP API listening interface",
+	}
 	SwarmPortFlag = cli.StringFlag{
 		Name:  "bzzport",
 		Usage: "Swarm local http api port",
@@ -112,19 +113,36 @@ var (
 		Name:  "defaultpath",
 		Usage: "path to file served for empty url path (none)",
 	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "force mime type",
+	}
 	CorsStringFlag = cli.StringFlag{
 		Name:  "corsdomain",
 		Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
 	}
 )
 
-func init() {
+var defaultNodeConfig = node.DefaultConfig
-	// Override flag defaults so bzzd can run alongside gexp.
-	utils.ListenPortFlag.Value = 30399
-	utils.IPCPathFlag.Value = utils.DirectoryString{Value: "bzzd.ipc"}
-	utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, swarmfs, web3"
 
-	// Set up the cli app.
+// This init function sets defaults so cmd/swarm can run alongside geth.
+func init() {
+	defaultNodeConfig.Name = clientIdentifier
+	defaultNodeConfig.Version = params.VersionWithCommit(gitCommit)
+	defaultNodeConfig.P2P.ListenAddr = ":30399"
+	defaultNodeConfig.IPCPath = "bzzd.ipc"
+	// Set flag defaults for --help display.
+	utils.ListenPortFlag.Value = 30399
+}
+
+var app = utils.NewApp(gitCommit, "Ethereum Swarm")
+
+// This init function creates the cli.App.
+func init() {
 	app.Action = bzzd
 	app.HideVersion = true // we have a command to print the version
 	app.Copyright = "Copyright 2013-2016 The go-ethereum / go-expanse Authors"
@@ -145,6 +163,15 @@ The output of this command is supposed to be machine-readable.
 			ArgsUsage: " <file>",
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
 `,
 		},
 		{
@@ -218,14 +245,15 @@ Cleans database of corrupted entries.
 		utils.MaxPeersFlag,
 		utils.NATFlag,
 		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
 		utils.IPCPathFlag,
+		utils.PasswordFileFlag,
 		// bzzd-specific flags
 		CorsStringFlag,
 		EthAPIFlag,
 		SwarmConfigPathFlag,
 		SwarmSwapEnabledFlag,
 		SwarmSyncEnabledFlag,
+		SwarmListenAddrFlag,
 		SwarmPortFlag,
 		SwarmAccountFlag,
 		SwarmNetworkIdFlag,
@@ -235,6 +263,8 @@ Cleans database of corrupted entries.
 		SwarmRecursiveUploadFlag,
 		SwarmWantManifestFlag,
 		SwarmUploadDefaultPath,
+		SwarmUpFromStdinFlag,
+		SwarmUploadMimeType,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Before = func(ctx *cli.Context) error {
@@ -256,7 +286,7 @@ func main() {
 
 func version(ctx *cli.Context) error {
 	fmt.Println(strings.Title(clientIdentifier))
-	fmt.Println("Version:", versionString)
+	fmt.Println("Version:", params.Version)
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
@@ -269,9 +299,16 @@ func version(ctx *cli.Context) error {
 }
 
 func bzzd(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	cfg := defaultNodeConfig
+	utils.SetNodeConfig(ctx, &cfg)
+	stack, err := node.New(&cfg)
+	if err != nil {
+		utils.Fatalf("can't create node: %v", err)
+	}
+
 	registerBzzService(ctx, stack)
 	utils.StartNode(stack)
+
 	go func() {
 		sigc := make(chan os.Signal, 1)
 		signal.Notify(sigc, syscall.SIGTERM)
@@ -280,6 +317,7 @@ func bzzd(ctx *cli.Context) error {
 		log.Info("Got sigterm, shutting swarm down...")
 		stack.Stop()
 	}()
+
 	networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
 	// Add bootnodes as initial peers.
 	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
@@ -296,7 +334,6 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
-
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -313,6 +350,9 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if len(bzzport) > 0 {
 		bzzconfig.Port = bzzport
 	}
+	if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" {
+		bzzconfig.ListenAddr = bzzaddr
+	}
 	swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
 	syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
 
@@ -326,6 +366,8 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 			if err != nil {
 				utils.Fatalf("Can't connect: %v", err)
 			}
+		} else {
+			swapEnabled = false
 		}
 		return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled, cors)
 	}
@@ -349,10 +391,10 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	am := stack.AccountManager()
 	ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
-	return decryptStoreAccount(ks, keyid)
+	return decryptStoreAccount(ks, keyid, utils.MakePasswordList(ctx))
 }
 
-func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKey {
+func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
 	var a accounts.Account
 	var err error
 	if common.IsHexAddress(account) {
@@ -373,9 +415,9 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	if err != nil {
 		utils.Fatalf("Can't load swarm account key: %v", err)
 	}
-	for i := 1; i <= 3; i++ {
+	for i := 0; i < 3; i++ {
-		passphrase := promptPassphrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i))
+		password := getPassPhrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i+1), i, passwords)
-		key, err := keystore.DecryptKey(keyjson, passphrase)
+		key, err := keystore.DecryptKey(keyjson, password)
 		if err == nil {
 			return key.PrivateKey
 		}
@@ -384,7 +426,18 @@ func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKe
 	return nil
 }
 
-func promptPassphrase(prompt string) string {
+// getPassPhrase retrieves the password associated with bzz account, either by fetching
+// from a list of pre-loaded passwords, or by requesting it interactively from user.
+func getPassPhrase(prompt string, i int, passwords []string) string {
+	// non-interactive
+	if len(passwords) > 0 {
+		if i < len(passwords) {
+			return passwords[i]
+		}
+		return passwords[len(passwords)-1]
+	}
+
+	// fallback to interactive mode
 	if prompt != "" {
 		fmt.Println(prompt)
 	}

cmd/swarm/manifest.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -25,6 +25,8 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
+	"github.com/expanse-org/go-expanse/swarm/api"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -41,7 +43,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 
 	if len(args) > 3 {
@@ -75,7 +77,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -105,7 +107,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        api.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -123,21 +125,17 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = &client{api: bzzapi}
+		client           = swarm.NewClient(bzzapi)
-		longestPathEntry = manifestEntry{
+		longestPathEntry = api.ManifestEntry{}
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.downloadManifest(hash)
+	_, err = client.DownloadManifest(hash)
 	if err != nil {
 		utils.Fatalf("Hash to add is not present: %v", err)
 	}
@@ -162,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -172,15 +170,15 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := manifestEntry{
+		newEntry := api.ManifestEntry{
-			Path:        path,
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -191,21 +189,13 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = &client{api: bzzapi}
+		client           = swarm.NewClient(bzzapi)
-		newEntry = manifestEntry{
+		newEntry         = api.ManifestEntry{}
-			Path:        "",
+		longestPathEntry = api.ManifestEntry{}
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -236,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -249,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := manifestEntry{
+				myEntry := api.ManifestEntry{
-					Path:        entry.Path,
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -265,7 +255,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -275,21 +265,13 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = &client{api: bzzapi}
+		client           = swarm.NewClient(bzzapi)
-		entryToRemove = manifestEntry{
+		entryToRemove    = api.ManifestEntry{}
-			Path:        "",
+		longestPathEntry = api.ManifestEntry{}
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = manifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -318,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -330,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -339,7 +321,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}

cmd/swarm/upload.go

@@ -18,8 +18,6 @@
 package main
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -32,58 +30,83 @@ import (
 	"strings"
 
 	"github.com/expanse-org/go-expanse/cmd/utils"
-	"github.com/expanse-org/go-expanse/log"
+	swarm "github.com/expanse-org/go-expanse/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
 func upload(ctx *cli.Context) {
+
 	args := ctx.Args()
 	var (
 		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		recursive    = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
+		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
+
 	if len(args) != 1 {
-		utils.Fatalf("Need filename as the first and only argument")
+		if fromStdin {
+			tmp, err := ioutil.TempFile("", "swarm-stdin")
+			if err != nil {
+				utils.Fatalf("error create tempfile: %s", err)
+			}
+			defer os.Remove(tmp.Name())
+			n, err := io.Copy(tmp, os.Stdin)
+			if err != nil {
+				utils.Fatalf("error copying stdin to tempfile: %s", err)
+			} else if n == 0 {
+				utils.Fatalf("error reading from stdin: zero length")
+			}
+			file = tmp.Name()
+		} else {
+			utils.Fatalf("Need filename as the first and only argument")
+		}
+	} else {
+		file = expandPath(args[0])
 	}
 
-	var (
+	if !wantManifest {
-		file   = args[0]
+		f, err := swarm.Open(file)
-		client = &client{api: bzzapi}
+		if err != nil {
-	)
+			utils.Fatalf("Error opening file: %s", err)
-	fi, err := os.Stat(expandPath(file))
+		}
-	if err != nil {
+		defer f.Close()
-		utils.Fatalf("Failed to stat file: %v", err)
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
+		return
 	}
-	if fi.IsDir() {
+
+	stat, err := os.Stat(file)
+	if err != nil {
+		utils.Fatalf("Error opening file: %s", err)
+	}
+	var hash string
+	if stat.IsDir() {
 		if !recursive {
 			utils.Fatalf("Argument is a directory and recursive upload is disabled")
 		}
-		if !wantManifest {
+		hash, err = client.UploadDirectory(file, defaultPath, "")
-			utils.Fatalf("Manifest is required for directory uploads")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
 		}
-		mhash, err := client.uploadDirectory(file, defaultPath)
+		f, err := swarm.Open(file)
 		if err != nil {
-			utils.Fatalf("Failed to upload directory: %v", err)
+			utils.Fatalf("Error opening file: %s", err)
 		}
-		fmt.Println(mhash)
+		defer f.Close()
-		return
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
 	}
-	entry, err := client.uploadFile(file, fi)
 	if err != nil {
-		utils.Fatalf("Upload failed: %v", err)
+		utils.Fatalf("Upload failed: %s", err)
-	}
-	mroot := manifest{[]manifestEntry{entry}}
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
-	hash, err := client.uploadManifest(mroot)
-	if err != nil {
-		utils.Fatalf("Manifest upload failed: %v", err)
 	}
 	fmt.Println(hash)
 }
@@ -112,147 +135,18 @@ func homeDir() string {
 	return ""
 }
 
-// client wraps interaction with the swarm HTTP gateway.
+func detectMimeType(file string) string {
-type client struct {
+	if ext := filepath.Ext(file); ext != "" {
-	api string
+		return mime.TypeByExtension(ext)
-}
+	}
-
+	f, err := os.Open(file)
-// manifest is the JSON representation of a swarm manifest.
+	if err != nil {
-type manifestEntry struct {
+		return ""
-	Hash        string `json:"hash,omitempty"`
+	}
-	ContentType string `json:"contentType,omitempty"`
+	defer f.Close()
-	Path        string `json:"path,omitempty"`
+	buf := make([]byte, 512)
-}
+	if n, _ := f.Read(buf); n > 0 {
-
+		return http.DetectContentType(buf)
-// manifest is the JSON representation of a swarm manifest.
+	}
-type manifest struct {
+	return ""
-	Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
-	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
-	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
-	mroot := manifest{}
-	req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
-	if err != nil {
-		return mroot, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return mroot, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-
-	err = json.Unmarshal(content, &mroot)
-	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
-	}
-	return mroot, err
 }

cmd/utils/customflags.go

@@ -17,6 +17,7 @@
 package utils
 
 import (
+	"encoding"
 	"errors"
 	"flag"
 	"fmt"
@@ -78,6 +79,58 @@ func (self DirectoryFlag) Apply(set *flag.FlagSet) {
 	})
 }
 
+type TextMarshaler interface {
+	encoding.TextMarshaler
+	encoding.TextUnmarshaler
+}
+
+// textMarshalerVal turns a TextMarshaler into a flag.Value
+type textMarshalerVal struct {
+	v TextMarshaler
+}
+
+func (v textMarshalerVal) String() string {
+	if v.v == nil {
+		return ""
+	}
+	text, _ := v.v.MarshalText()
+	return string(text)
+}
+
+func (v textMarshalerVal) Set(s string) error {
+	return v.v.UnmarshalText([]byte(s))
+}
+
+// TextMarshalerFlag wraps a TextMarshaler value.
+type TextMarshalerFlag struct {
+	Name  string
+	Value TextMarshaler
+	Usage string
+}
+
+func (f TextMarshalerFlag) GetName() string {
+	return f.Name
+}
+
+func (f TextMarshalerFlag) String() string {
+	return fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+}
+
+func (f TextMarshalerFlag) Apply(set *flag.FlagSet) {
+	eachName(f.Name, func(name string) {
+		set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage)
+	})
+}
+
+// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set.
+func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler {
+	val := ctx.GlobalGeneric(name)
+	if val == nil {
+		return nil
+	}
+	return val.(textMarshalerVal).v
+}
+
 // BigFlag is a command line flag that accepts 256 bit big integers in decimal or
 // hexadecimal syntax.
 type BigFlag struct {

cmd/wnode/main.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -27,7 +27,9 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -46,7 +48,6 @@ import (
 )
 
 const quitCommand = "~Q"
-const symKeyName = "da919ea33001b04dfc630522e33078ec0df11"
 
 // singletons
 var (
@@ -65,6 +66,7 @@ var (
 	asymKey    *ecdsa.PrivateKey
 	nodeid     *ecdsa.PrivateKey
 	topic      whisper.TopicType
+	asymKeyID  string
 	filterID   string
 	symPass    string
 	msPassword string
@@ -72,27 +74,30 @@ var (
 
 // cmd arguments
 var (
-	echoMode       = flag.Bool("e", false, "echo mode: prints some arguments for diagnostics")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
-	bootstrapMode  = flag.Bool("b", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
-	forwarderMode  = flag.Bool("f", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
-	mailServerMode = flag.Bool("s", false, "mail server mode: delivers expired messages on demand")
+	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
-	requestMail    = flag.Bool("r", false, "request expired messages from the bootstrap server")
+	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
-	asymmetricMode = flag.Bool("a", false, "use asymmetric encryption")
+	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
-	testMode       = flag.Bool("t", false, "use of predefined parameters for diagnostics")
+	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
-	generateKey    = flag.Bool("k", false, "generate and show the private key")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
-	argVerbosity = flag.Int("verbosity", int(log.LvlWarn), "log verbosity level")
+	argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level")
 	argTTL       = flag.Uint("ttl", 30, "time-to-live for messages in seconds")
 	argWorkTime  = flag.Uint("work", 5, "work time in seconds")
-	argPoW       = flag.Float64("pow", whisper.MinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argMaxSize   = flag.Int("maxsize", whisper.DefaultMaxMessageLength, "max size of message")
-	argServerPoW = flag.Float64("mspow", whisper.MinimumPoW, "PoW requirement for Mail Server request")
+	argPoW       = flag.Float64("pow", whisper.DefaultMinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argServerPoW = flag.Float64("mspow", whisper.DefaultMinimumPoW, "PoW requirement for Mail Server request")
 
-	argIP     = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:42786)")
+	argIP      = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:42786)")
-	argPub    = flag.String("pub", "", "public key for asymmetric encryption")
+	argPub     = flag.String("pub", "", "public key for asymmetric encryption")
-	argDBPath = flag.String("dbpath", "", "path to the server's DB directory")
+	argDBPath  = flag.String("dbpath", "", "path to the server's DB directory")
-	argIDFile = flag.String("idfile", "", "file name with node id (private key)")
+	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
-	argEnode  = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
+	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
-	argTopic  = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4defaced)")
+	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
 )
 
 func main() {
@@ -134,6 +139,14 @@ func processArgs() {
 		}
 	}
 
+	if len(*argSaveDir) > 0 {
+		if _, err := os.Stat(*argSaveDir); os.IsNotExist(err) {
+			utils.Fatalf("Download directory '%s' does not exist", *argSaveDir)
+		}
+	} else if *fileExMode {
+		utils.Fatalf("Parameter 'savedir' is mandatory for file exchange mode")
+	}
+
 	if *echoMode {
 		echo()
 	}
@@ -170,7 +183,7 @@ func initialize() {
 
 	if *testMode {
 		symPass = "wwww" // ascii code: 0x77777777
-		msPassword = "mail server test password"
+		msPassword = "wwww"
 	}
 
 	if *bootstrapMode {
@@ -199,9 +212,40 @@ func initialize() {
 		shh = whisper.New()
 	}
 
-	asymKey = shh.NewIdentity()
+	if *argPoW != whisper.DefaultMinimumPoW {
+		err := shh.SetMinimumPoW(*argPoW)
+		if err != nil {
+			utils.Fatalf("Failed to set PoW: %s", err)
+		}
+	}
+
+	if *argMaxSize != whisper.DefaultMaxMessageLength {
+		err := shh.SetMaxMessageLength(*argMaxSize)
+		if err != nil {
+			utils.Fatalf("Failed to set max message size: %s", err)
+		}
+	}
+
+	asymKeyID, err = shh.NewKeyPair()
+	if err != nil {
+		utils.Fatalf("Failed to generate a new key pair: %s", err)
+	}
+
+	asymKey, err = shh.GetPrivateKey(asymKeyID)
+	if err != nil {
+		utils.Fatalf("Failed to retrieve a new key pair: %s", err)
+	}
+
 	if nodeid == nil {
-		nodeid = shh.NewIdentity()
+		tmpID, err := shh.NewKeyPair()
+		if err != nil {
+			utils.Fatalf("Failed to generate a new key pair: %s", err)
+		}
+
+		nodeid, err = shh.GetPrivateKey(tmpID)
+		if err != nil {
+			utils.Fatalf("Failed to retrieve a new key pair: %s", err)
+		}
 	}
 
 	maxPeers := 80
@@ -213,7 +257,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("whisper-go", "5.0"),
+			Name:           common.MakeName("wnode", "5.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -263,7 +307,11 @@ func configureNode() {
 	if *asymmetricMode {
 		if len(*argPub) == 0 {
 			s := scanLine("Please enter the peer's public key: ")
-			pub = crypto.ToECDSAPub(common.FromHex(s))
+			b := common.FromHex(s)
+			if b == nil {
+				utils.Fatalf("Error: can not convert hexadecimal string")
+			}
+			pub = crypto.ToECDSAPub(b)
 			if !isKeyValid(pub) {
 				utils.Fatalf("Error: invalid public key")
 			}
@@ -282,17 +330,25 @@ func configureNode() {
 
 	if !*asymmetricMode && !*forwarderMode {
 		if len(symPass) == 0 {
-			symPass, err = console.Stdin.PromptPassword("Please enter the password: ")
+			symPass, err = console.Stdin.PromptPassword("Please enter the password for symmetric encryption: ")
 			if err != nil {
 				utils.Fatalf("Failed to read passphrase: %v", err)
 			}
 		}
 
-		shh.AddSymKey(symKeyName, []byte(symPass))
+		symKeyID, err := shh.AddSymKeyFromPassword(symPass)
-		symKey = shh.GetSymKey(symKeyName)
+		if err != nil {
+			utils.Fatalf("Failed to create symmetric key: %s", err)
+		}
+		symKey, err = shh.GetSymKey(symKeyID)
+		if err != nil {
+			utils.Fatalf("Failed to save symmetric key: %s", err)
+		}
 		if len(*argTopic) == 0 {
 			generateTopic([]byte(symPass))
 		}
+
+		fmt.Printf("Filter is configured for the topic: %x \n", topic)
 	}
 
 	if *mailServerMode {
@@ -302,20 +358,19 @@ func configureNode() {
 	}
 
 	filter := whisper.Filter{
-		KeySym:    symKey,
+		KeySym:   symKey,
-		KeyAsym:   asymKey,
+		KeyAsym:  asymKey,
-		Topics:    []whisper.TopicType{topic},
+		Topics:   [][]byte{topic[:]},
-		AcceptP2P: p2pAccept,
+		AllowP2P: p2pAccept,
 	}
-	filterID, err = shh.Watch(&filter)
+	filterID, err = shh.Subscribe(&filter)
 	if err != nil {
 		utils.Fatalf("Failed to install filter: %s", err)
 	}
-	fmt.Printf("Filter is configured for the topic: %x \n", topic)
 }
 
 func generateTopic(password []byte) {
-	x := pbkdf2.Key(password, password, 8196, 128, sha512.New)
+	x := pbkdf2.Key(password, password, 4096, 128, sha512.New)
 	for i := 0; i < len(x); i++ {
 		topic[i%whisper.TopicLength] ^= x[i]
 	}
@@ -351,6 +406,8 @@ func run() {
 
 	if *requestMail {
 		requestExpiredMessagesLoop()
+	} else if *fileExMode {
+		sendFilesLoop()
 	} else {
 		sendLoop()
 	}
@@ -376,6 +433,31 @@ func sendLoop() {
 	}
 }
 
+func sendFilesLoop() {
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			close(done)
+			break
+		}
+		b, err := ioutil.ReadFile(s)
+		if err != nil {
+			fmt.Printf(">>> Error: %s \n", err)
+			continue
+		} else {
+			h := sendMsg(b)
+			if (h == common.Hash{}) {
+				fmt.Printf(">>> Error: message was not sent \n")
+			} else {
+				timestamp := time.Now().Unix()
+				from := crypto.PubkeyToAddress(asymKey.PublicKey)
+				fmt.Printf("\n%d <%x>: sent message with hash %x\n", timestamp, from, h)
+			}
+		}
+	}
+}
+
 func scanLine(prompt string) string {
 	if len(prompt) > 0 {
 		fmt.Print(prompt)
@@ -402,7 +484,7 @@ func scanUint(prompt string) uint32 {
 	return uint32(i)
 }
 
-func sendMsg(payload []byte) {
+func sendMsg(payload []byte) common.Hash {
 	params := whisper.MessageParams{
 		Src:      asymKey,
 		Dst:      pub,
@@ -414,17 +496,23 @@ func sendMsg(payload []byte) {
 		WorkTime: uint32(*argWorkTime),
 	}
 
-	msg := whisper.NewSentMessage(&params)
+	msg, err := whisper.NewSentMessage(&params)
+	if err != nil {
+		utils.Fatalf("failed to create new message: %s", err)
+	}
 	envelope, err := msg.Wrap(&params)
 	if err != nil {
 		fmt.Printf("failed to seal message: %v \n", err)
-		return
+		return common.Hash{}
 	}
 
 	err = shh.Send(envelope)
 	if err != nil {
 		fmt.Printf("failed to send message: %v \n", err)
+		return common.Hash{}
 	}
+
+	return envelope.Hash()
 }
 
 func messageLoop() {
@@ -440,7 +528,11 @@ func messageLoop() {
 		case <-ticker.C:
 			messages := f.Retrieve()
 			for _, msg := range messages {
-				printMessageInfo(msg)
+				if *fileExMode || len(msg.Payload) > 2048 {
+					writeMessageToFile(*argSaveDir, msg)
+				} else {
+					printMessageInfo(msg)
+				}
 			}
 		case <-done:
 			return
@@ -464,19 +556,47 @@ func printMessageInfo(msg *whisper.ReceivedMessage) {
 	}
 }
 
+func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
+	timestamp := fmt.Sprintf("%d", msg.Sent)
+	name := fmt.Sprintf("%x", msg.EnvelopeHash)
+
+	var address common.Address
+	if msg.Src != nil {
+		address = crypto.PubkeyToAddress(*msg.Src)
+	}
+
+	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+		// message from myself: don't save, only report
+		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
+	} else if len(dir) > 0 {
+		fullpath := filepath.Join(dir, name)
+		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		if err != nil {
+			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
+		} else {
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+		}
+	} else {
+		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+	}
+}
+
 func requestExpiredMessagesLoop() {
 	var key, peerID []byte
 	var timeLow, timeUpp uint32
 	var t string
 	var xt, empty whisper.TopicType
 
-	err := shh.AddSymKey(mailserver.MailServerKeyName, []byte(msPassword))
+	keyID, err := shh.AddSymKeyFromPassword(msPassword)
 	if err != nil {
 		utils.Fatalf("Failed to create symmetric key for mail request: %s", err)
 	}
-	key = shh.GetSymKey(mailserver.MailServerKeyName)
+	key, err = shh.GetSymKey(keyID)
+	if err != nil {
+		utils.Fatalf("Failed to save symmetric key for mail request: %s", err)
+	}
 	peerID = extractIdFromEnode(*argEnode)
-	shh.MarkPeerTrusted(peerID)
+	shh.AllowP2PMessagesFromPeer(peerID)
 
 	for {
 		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
@@ -508,7 +628,10 @@ func requestExpiredMessagesLoop() {
 		params.Src = nodeid
 		params.WorkTime = 5
 
-		msg := whisper.NewSentMessage(&params)
+		msg, err := whisper.NewSentMessage(&params)
+		if err != nil {
+			utils.Fatalf("failed to create new message: %s", err)
+		}
 		env, err := msg.Wrap(&params)
 		if err != nil {
 			utils.Fatalf("Wrap failed: %s", err)
@@ -527,7 +650,6 @@ func extractIdFromEnode(s string) []byte {
 	n, err := discover.ParseNode(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)
-		return nil
 	}
 	return n.ID[:]
 }

common/bitutil/bitutil.go

@@ -0,0 +1,188 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Adapted from: https://golang.org/src/crypto/cipher/xor.go
+
+// Package bitutil implements fast bitwise operations.
+package bitutil
+
+import (
+	"runtime"
+	"unsafe"
+)
+
+const wordSize = int(unsafe.Sizeof(uintptr(0)))
+const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x"
+
+// XORBytes xors the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes xor'd.
+func XORBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastXORBytes(dst, a, b)
+	}
+	return safeXORBytes(dst, a, b)
+}
+
+// fastXORBytes xors in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastXORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] ^ bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] ^ b[i]
+	}
+	return n
+}
+
+// safeXORBytes xors one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeXORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] ^ b[i]
+	}
+	return n
+}
+
+// ANDBytes ands the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes and'd.
+func ANDBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastANDBytes(dst, a, b)
+	}
+	return safeANDBytes(dst, a, b)
+}
+
+// fastANDBytes ands in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastANDBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] & bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] & b[i]
+	}
+	return n
+}
+
+// safeANDBytes ands one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeANDBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] & b[i]
+	}
+	return n
+}
+
+// ORBytes ors the bytes in a and b. The destination is assumed to have enough
+// space. Returns the number of bytes or'd.
+func ORBytes(dst, a, b []byte) int {
+	if supportsUnaligned {
+		return fastORBytes(dst, a, b)
+	}
+	return safeORBytes(dst, a, b)
+}
+
+// fastORBytes ors in bulk. It only works on architectures that support
+// unaligned read/writes.
+func fastORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	w := n / wordSize
+	if w > 0 {
+		dw := *(*[]uintptr)(unsafe.Pointer(&dst))
+		aw := *(*[]uintptr)(unsafe.Pointer(&a))
+		bw := *(*[]uintptr)(unsafe.Pointer(&b))
+		for i := 0; i < w; i++ {
+			dw[i] = aw[i] | bw[i]
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		dst[i] = a[i] | b[i]
+	}
+	return n
+}
+
+// safeORBytes ors one by one. It works on all architectures, independent if
+// it supports unaligned read/writes or not.
+func safeORBytes(dst, a, b []byte) int {
+	n := len(a)
+	if len(b) < n {
+		n = len(b)
+	}
+	for i := 0; i < n; i++ {
+		dst[i] = a[i] | b[i]
+	}
+	return n
+}
+
+// TestBytes tests whether any bit is set in the input byte slice.
+func TestBytes(p []byte) bool {
+	if supportsUnaligned {
+		return fastTestBytes(p)
+	}
+	return safeTestBytes(p)
+}
+
+// fastTestBytes tests for set bits in bulk. It only works on architectures that
+// support unaligned read/writes.
+func fastTestBytes(p []byte) bool {
+	n := len(p)
+	w := n / wordSize
+	if w > 0 {
+		pw := *(*[]uintptr)(unsafe.Pointer(&p))
+		for i := 0; i < w; i++ {
+			if pw[i] != 0 {
+				return true
+			}
+		}
+	}
+	for i := (n - n%wordSize); i < n; i++ {
+		if p[i] != 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// safeTestBytes tests for set bits one byte at a time. It works on all
+// architectures, independent if it supports unaligned read/writes or not.
+func safeTestBytes(p []byte) bool {
+	for i := 0; i < len(p); i++ {
+		if p[i] != 0 {
+			return true
+		}
+	}
+	return false
+}

common/bitutil/bitutil_test.go

@@ -0,0 +1,215 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go
+
+package bitutil
+
+import (
+	"bytes"
+	"testing"
+)
+
+// Tests that bitwise XOR works for various alignments.
+func TestXOR(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				XORBytes(d1, p, q)
+				safeXORBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal", d1, d2)
+				}
+			}
+		}
+	}
+}
+
+// Tests that bitwise AND works for various alignments.
+func TestAND(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				ANDBytes(d1, p, q)
+				safeANDBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal")
+				}
+			}
+		}
+	}
+}
+
+// Tests that bitwise OR works for various alignments.
+func TestOR(t *testing.T) {
+	for alignP := 0; alignP < 2; alignP++ {
+		for alignQ := 0; alignQ < 2; alignQ++ {
+			for alignD := 0; alignD < 2; alignD++ {
+				p := make([]byte, 1023)[alignP:]
+				q := make([]byte, 1023)[alignQ:]
+
+				for i := 0; i < len(p); i++ {
+					p[i] = byte(i)
+				}
+				for i := 0; i < len(q); i++ {
+					q[i] = byte(len(q) - i)
+				}
+				d1 := make([]byte, 1023+alignD)[alignD:]
+				d2 := make([]byte, 1023+alignD)[alignD:]
+
+				ORBytes(d1, p, q)
+				safeORBytes(d2, p, q)
+				if !bytes.Equal(d1, d2) {
+					t.Error("not equal")
+				}
+			}
+		}
+	}
+}
+
+// Tests that bit testing works for various alignments.
+func TestTest(t *testing.T) {
+	for align := 0; align < 2; align++ {
+		// Test for bits set in the bulk part
+		p := make([]byte, 1023)[align:]
+		p[100] = 1
+
+		if TestBytes(p) != safeTestBytes(p) {
+			t.Error("not equal")
+		}
+		// Test for bits set in the tail part
+		q := make([]byte, 1023)[align:]
+		q[len(q)-1] = 1
+
+		if TestBytes(q) != safeTestBytes(q) {
+			t.Error("not equal")
+		}
+	}
+}
+
+// Benchmarks the potentially optimized XOR performance.
+func BenchmarkFastXOR1KB(b *testing.B) { benchmarkFastXOR(b, 1024) }
+func BenchmarkFastXOR2KB(b *testing.B) { benchmarkFastXOR(b, 2048) }
+func BenchmarkFastXOR4KB(b *testing.B) { benchmarkFastXOR(b, 4096) }
+
+func benchmarkFastXOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		XORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline XOR performance.
+func BenchmarkBaseXOR1KB(b *testing.B) { benchmarkBaseXOR(b, 1024) }
+func BenchmarkBaseXOR2KB(b *testing.B) { benchmarkBaseXOR(b, 2048) }
+func BenchmarkBaseXOR4KB(b *testing.B) { benchmarkBaseXOR(b, 4096) }
+
+func benchmarkBaseXOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeXORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized AND performance.
+func BenchmarkFastAND1KB(b *testing.B) { benchmarkFastAND(b, 1024) }
+func BenchmarkFastAND2KB(b *testing.B) { benchmarkFastAND(b, 2048) }
+func BenchmarkFastAND4KB(b *testing.B) { benchmarkFastAND(b, 4096) }
+
+func benchmarkFastAND(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		ANDBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline AND performance.
+func BenchmarkBaseAND1KB(b *testing.B) { benchmarkBaseAND(b, 1024) }
+func BenchmarkBaseAND2KB(b *testing.B) { benchmarkBaseAND(b, 2048) }
+func BenchmarkBaseAND4KB(b *testing.B) { benchmarkBaseAND(b, 4096) }
+
+func benchmarkBaseAND(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeANDBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized OR performance.
+func BenchmarkFastOR1KB(b *testing.B) { benchmarkFastOR(b, 1024) }
+func BenchmarkFastOR2KB(b *testing.B) { benchmarkFastOR(b, 2048) }
+func BenchmarkFastOR4KB(b *testing.B) { benchmarkFastOR(b, 4096) }
+
+func benchmarkFastOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		ORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the baseline OR performance.
+func BenchmarkBaseOR1KB(b *testing.B) { benchmarkBaseOR(b, 1024) }
+func BenchmarkBaseOR2KB(b *testing.B) { benchmarkBaseOR(b, 2048) }
+func BenchmarkBaseOR4KB(b *testing.B) { benchmarkBaseOR(b, 4096) }
+
+func benchmarkBaseOR(b *testing.B, size int) {
+	p, q := make([]byte, size), make([]byte, size)
+
+	for i := 0; i < b.N; i++ {
+		safeORBytes(p, p, q)
+	}
+}
+
+// Benchmarks the potentially optimized bit testing performance.
+func BenchmarkFastTest1KB(b *testing.B) { benchmarkFastTest(b, 1024) }
+func BenchmarkFastTest2KB(b *testing.B) { benchmarkFastTest(b, 2048) }
+func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) }
+
+func benchmarkFastTest(b *testing.B, size int) {
+	p := make([]byte, size)
+	for i := 0; i < b.N; i++ {
+		TestBytes(p)
+	}
+}
+
+// Benchmarks the baseline bit testing performance.
+func BenchmarkBaseTest1KB(b *testing.B) { benchmarkBaseTest(b, 1024) }
+func BenchmarkBaseTest2KB(b *testing.B) { benchmarkBaseTest(b, 2048) }
+func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) }
+
+func benchmarkBaseTest(b *testing.B, size int) {
+	p := make([]byte, size)
+	for i := 0; i < b.N; i++ {
+		safeTestBytes(p)
+	}
+}

common/bitutil/compress.go

@@ -0,0 +1,170 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bitutil
+
+import "errors"
+
+var (
+	// errMissingData is returned from decompression if the byte referenced by
+	// the bitset header overflows the input data.
+	errMissingData = errors.New("missing bytes on input")
+
+	// errUnreferencedData is returned from decompression if not all bytes were used
+	// up from the input data after decompressing it.
+	errUnreferencedData = errors.New("extra bytes on input")
+
+	// errExceededTarget is returned from decompression if the bitset header has
+	// more bits defined than the number of target buffer space available.
+	errExceededTarget = errors.New("target data size exceeded")
+
+	// errZeroContent is returned from decompression if a data byte referenced in
+	// the bitset header is actually a zero byte.
+	errZeroContent = errors.New("zero byte in input content")
+)
+
+// The compression algorithm implemented by CompressBytes and DecompressBytes is
+// optimized for sparse input data which contains a lot of zero bytes. Decompression
+// requires knowledge of the decompressed data length.
+//
+// Compression works as follows:
+//
+//   if data only contains zeroes,
+//       CompressBytes(data) == nil
+//   otherwise if len(data) <= 1,
+//       CompressBytes(data) == data
+//   otherwise:
+//       CompressBytes(data) == append(CompressBytes(nonZeroBitset(data)), nonZeroBytes(data)...)
+//       where
+//         nonZeroBitset(data) is a bit vector with len(data) bits (MSB first):
+//             nonZeroBitset(data)[i/8] && (1 << (7-i%8)) != 0  if data[i] != 0
+//             len(nonZeroBitset(data)) == (len(data)+7)/8
+//         nonZeroBytes(data) contains the non-zero bytes of data in the same order
+
+// CompressBytes compresses the input byte slice according to the sparse bitset
+// representation algorithm. If the result is bigger than the original input, no
+// compression is done.
+func CompressBytes(data []byte) []byte {
+	if out := bitsetEncodeBytes(data); len(out) < len(data) {
+		return out
+	}
+	cpy := make([]byte, len(data))
+	copy(cpy, data)
+	return cpy
+}
+
+// bitsetEncodeBytes compresses the input byte slice according to the sparse
+// bitset representation algorithm.
+func bitsetEncodeBytes(data []byte) []byte {
+	// Empty slices get compressed to nil
+	if len(data) == 0 {
+		return nil
+	}
+	// One byte slices compress to nil or retain the single byte
+	if len(data) == 1 {
+		if data[0] == 0 {
+			return nil
+		}
+		return data
+	}
+	// Calculate the bitset of set bytes, and gather the non-zero bytes
+	nonZeroBitset := make([]byte, (len(data)+7)/8)
+	nonZeroBytes := make([]byte, 0, len(data))
+
+	for i, b := range data {
+		if b != 0 {
+			nonZeroBytes = append(nonZeroBytes, b)
+			nonZeroBitset[i/8] |= 1 << byte(7-i%8)
+		}
+	}
+	if len(nonZeroBytes) == 0 {
+		return nil
+	}
+	return append(bitsetEncodeBytes(nonZeroBitset), nonZeroBytes...)
+}
+
+// DecompressBytes decompresses data with a known target size. If the input data
+// matches the size of the target, it means no compression was done in the first
+// place.
+func DecompressBytes(data []byte, target int) ([]byte, error) {
+	if len(data) > target {
+		return nil, errExceededTarget
+	}
+	if len(data) == target {
+		cpy := make([]byte, len(data))
+		copy(cpy, data)
+		return cpy, nil
+	}
+	return bitsetDecodeBytes(data, target)
+}
+
+// bitsetDecodeBytes decompresses data with a known target size.
+func bitsetDecodeBytes(data []byte, target int) ([]byte, error) {
+	out, size, err := bitsetDecodePartialBytes(data, target)
+	if err != nil {
+		return nil, err
+	}
+	if size != len(data) {
+		return nil, errUnreferencedData
+	}
+	return out, nil
+}
+
+// bitsetDecodePartialBytes decompresses data with a known target size, but does
+// not enforce consuming all the input bytes. In addition to the decompressed
+// output, the function returns the length of compressed input data corresponding
+// to the output as the input slice may be longer.
+func bitsetDecodePartialBytes(data []byte, target int) ([]byte, int, error) {
+	// Sanity check 0 targets to avoid infinite recursion
+	if target == 0 {
+		return nil, 0, nil
+	}
+	// Handle the zero and single byte corner cases
+	decomp := make([]byte, target)
+	if len(data) == 0 {
+		return decomp, 0, nil
+	}
+	if target == 1 {
+		decomp[0] = data[0] // copy to avoid referencing the input slice
+		if data[0] != 0 {
+			return decomp, 1, nil
+		}
+		return decomp, 0, nil
+	}
+	// Decompress the bitset of set bytes and distribute the non zero bytes
+	nonZeroBitset, ptr, err := bitsetDecodePartialBytes(data, (target+7)/8)
+	if err != nil {
+		return nil, ptr, err
+	}
+	for i := 0; i < 8*len(nonZeroBitset); i++ {
+		if nonZeroBitset[i/8]&(1<<byte(7-i%8)) != 0 {
+			// Make sure we have enough data to push into the correct slot
+			if ptr >= len(data) {
+				return nil, 0, errMissingData
+			}
+			if i >= len(decomp) {
+				return nil, 0, errExceededTarget
+			}
+			// Make sure the data is valid and push into the slot
+			if data[ptr] == 0 {
+				return nil, 0, errZeroContent
+			}
+			decomp[i] = data[ptr]
+			ptr++
+		}
+	}
+	return decomp, ptr, nil
+}

common/bitutil/compress_fuzz.go

@@ -0,0 +1,56 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build gofuzz
+
+package bitutil
+
+import "bytes"
+
+// Fuzz implements a go-fuzz fuzzer method to test various encoding method
+// invocations.
+func Fuzz(data []byte) int {
+	if len(data) == 0 {
+		return -1
+	}
+	if data[0]%2 == 0 {
+		return fuzzEncode(data[1:])
+	}
+	return fuzzDecode(data[1:])
+}
+
+// fuzzEncode implements a go-fuzz fuzzer method to test the bitset encoding and
+// decoding algorithm.
+func fuzzEncode(data []byte) int {
+	proc, _ := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data))
+	if !bytes.Equal(data, proc) {
+		panic("content mismatch")
+	}
+	return 0
+}
+
+// fuzzDecode implements a go-fuzz fuzzer method to test the bit decoding and
+// reencoding algorithm.
+func fuzzDecode(data []byte) int {
+	blob, err := bitsetDecodeBytes(data, 1024)
+	if err != nil {
+		return 0
+	}
+	if comp := bitsetEncodeBytes(blob); !bytes.Equal(comp, data) {
+		panic("content mismatch")
+	}
+	return 0
+}

common/bitutil/compress_test.go

@@ -0,0 +1,181 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bitutil
+
+import (
+	"bytes"
+	"math/rand"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/common/hexutil"
+)
+
+// Tests that data bitset encoding and decoding works and is bijective.
+func TestEncodingCycle(t *testing.T) {
+	tests := []string{
+		// Tests generated by go-fuzz to maximize code coverage
+		"0x000000000000000000",
+		"0xef0400",
+		"0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb",
+		"0x7b64000000",
+		"0x000034000000000000",
+		"0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f0000000000000000000",
+		"0x4912385c0e7b64000000",
+		"0x000034000000000000000000000000000000",
+		"0x00",
+		"0x000003e834ff7f0000",
+		"0x0000",
+		"0x0000000000000000000000000000000000000000000000000000000000ff00",
+		"0x895f0c6a020f850c6a020f85f88df88d",
+		"0xdf7070533534333636313639343638373432313536346c1bc3315aac2f65fefb",
+		"0x0000000000",
+		"0xdf70706336346c65fefb",
+		"0x00006d643634000000",
+		"0xdf7070533534333636313639343638373532313536346c1bc333393438373130707063363430353639343638373532313536346c1bc333393438336336346c65fe",
+	}
+	for i, tt := range tests {
+		data := hexutil.MustDecode(tt)
+
+		proc, err := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data))
+		if err != nil {
+			t.Errorf("test %d: failed to decompress compressed data: %v", i, err)
+			continue
+		}
+		if !bytes.Equal(data, proc) {
+			t.Errorf("test %d: compress/decompress mismatch: have %x, want %x", i, proc, data)
+		}
+	}
+}
+
+// Tests that data bitset decoding and rencoding works and is bijective.
+func TestDecodingCycle(t *testing.T) {
+	tests := []struct {
+		size  int
+		input string
+		fail  error
+	}{
+		{size: 0, input: "0x"},
+
+		// Crashers generated by go-fuzz
+		{size: 0, input: "0x0020", fail: errUnreferencedData},
+		{size: 0, input: "0x30", fail: errUnreferencedData},
+		{size: 1, input: "0x00", fail: errUnreferencedData},
+		{size: 2, input: "0x07", fail: errMissingData},
+		{size: 1024, input: "0x8000", fail: errZeroContent},
+
+		// Tests generated by go-fuzz to maximize code coverage
+		{size: 29490, input: "0x343137343733323134333839373334323073333930783e3078333930783e70706336346c65303e", fail: errMissingData},
+		{size: 59395, input: "0x00", fail: errUnreferencedData},
+		{size: 52574, input: "0x70706336346c65c0de", fail: errExceededTarget},
+		{size: 42264, input: "0x07", fail: errMissingData},
+		{size: 52, input: "0xa5045bad48f4", fail: errExceededTarget},
+		{size: 52574, input: "0xc0de", fail: errMissingData},
+		{size: 52574, input: "0x"},
+		{size: 29490, input: "0x34313734373332313433383937333432307333393078073034333839373334323073333930783e3078333937333432307333393078073061333930783e70706336346c65303e", fail: errMissingData},
+		{size: 29491, input: "0x3973333930783e30783e", fail: errMissingData},
+
+		{size: 1024, input: "0x808080608080"},
+		{size: 1024, input: "0x808470705e3632383337363033313434303137393130306c6580ef46806380635a80"},
+		{size: 1024, input: "0x8080808070"},
+		{size: 1024, input: "0x808070705e36346c6580ef46806380635a80"},
+		{size: 1024, input: "0x80808046802680"},
+		{size: 1024, input: "0x4040404035"},
+		{size: 1024, input: "0x4040bf3ba2b3f684402d353234373438373934409fe5b1e7ada94ebfd7d0505e27be4035"},
+		{size: 1024, input: "0x404040bf3ba2b3f6844035"},
+		{size: 1024, input: "0x40402d35323437343837393440bfd7d0505e27be4035"},
+	}
+	for i, tt := range tests {
+		data := hexutil.MustDecode(tt.input)
+
+		orig, err := bitsetDecodeBytes(data, tt.size)
+		if err != tt.fail {
+			t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.fail)
+		}
+		if err != nil {
+			continue
+		}
+		if comp := bitsetEncodeBytes(orig); !bytes.Equal(comp, data) {
+			t.Errorf("test %d: decompress/compress mismatch: have %x, want %x", i, comp, data)
+		}
+	}
+}
+
+// TestCompression tests that compression works by returning either the bitset
+// encoded input, or the actual input if the bitset version is longer.
+func TestCompression(t *testing.T) {
+	// Check the the compression returns the bitset encoding is shorter
+	in := hexutil.MustDecode("0x4912385c0e7b64000000")
+	out := hexutil.MustDecode("0x80fe4912385c0e7b64")
+
+	if data := CompressBytes(in); bytes.Compare(data, out) != 0 {
+		t.Errorf("encoding mismatch for sparse data: have %x, want %x", data, out)
+	}
+	if data, err := DecompressBytes(out, len(in)); err != nil || bytes.Compare(data, in) != 0 {
+		t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err)
+	}
+	// Check the the compression returns the input if the bitset encoding is longer
+	in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")
+	out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")
+
+	if data := CompressBytes(in); bytes.Compare(data, out) != 0 {
+		t.Errorf("encoding mismatch for dense data: have %x, want %x", data, out)
+	}
+	if data, err := DecompressBytes(out, len(in)); err != nil || bytes.Compare(data, in) != 0 {
+		t.Errorf("decoding mismatch for dense data: have %x, want %x, error %v", data, in, err)
+	}
+	// Check that decompressing a longer input than the target fails
+	if _, err := DecompressBytes([]byte{0xc0, 0x01, 0x01}, 2); err != errExceededTarget {
+		t.Errorf("decoding error mismatch for long data: have %v, want %v", err, errExceededTarget)
+	}
+}
+
+// Crude benchmark for compressing random slices of bytes.
+func BenchmarkEncoding1KBVerySparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.0001) }
+func BenchmarkEncoding2KBVerySparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.0001) }
+func BenchmarkEncoding4KBVerySparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.0001) }
+
+func BenchmarkEncoding1KBSparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.001) }
+func BenchmarkEncoding2KBSparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.001) }
+func BenchmarkEncoding4KBSparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.001) }
+
+func BenchmarkEncoding1KBDense(b *testing.B) { benchmarkEncoding(b, 1024, 0.1) }
+func BenchmarkEncoding2KBDense(b *testing.B) { benchmarkEncoding(b, 2048, 0.1) }
+func BenchmarkEncoding4KBDense(b *testing.B) { benchmarkEncoding(b, 4096, 0.1) }
+
+func BenchmarkEncoding1KBSaturated(b *testing.B) { benchmarkEncoding(b, 1024, 0.5) }
+func BenchmarkEncoding2KBSaturated(b *testing.B) { benchmarkEncoding(b, 2048, 0.5) }
+func BenchmarkEncoding4KBSaturated(b *testing.B) { benchmarkEncoding(b, 4096, 0.5) }
+
+func benchmarkEncoding(b *testing.B, bytes int, fill float64) {
+	// Generate a random slice of bytes to compress
+	random := rand.NewSource(0) // reproducible and comparable
+
+	data := make([]byte, bytes)
+	bits := int(float64(bytes) * 8 * fill)
+
+	for i := 0; i < bits; i++ {
+		idx := random.Int63() % int64(len(data))
+		bit := uint(random.Int63() % 8)
+		data[idx] |= 1 << bit
+	}
+	// Reset the benchmark and measure encoding/decoding
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		bitsetDecodeBytes(bitsetEncodeBytes(data), len(data))
+	}
+}

common/bytes.go

@@ -89,18 +89,18 @@ func Hex2BytesFixed(str string, flen int) []byte {
 }
 
 func RightPadBytes(slice []byte, l int) []byte {
-	if l < len(slice) {
+	if l <= len(slice) {
 		return slice
 	}
 
 	padded := make([]byte, l)
-	copy(padded[0:len(slice)], slice)
+	copy(padded, slice)
 
 	return padded
 }
 
 func LeftPadBytes(slice []byte, l int) []byte {
-	if l < len(slice) {
+	if l <= len(slice) {
 		return slice
 	}
 

common/compiler/solidity.go

@@ -25,20 +25,11 @@ import (
 	"io/ioutil"
 	"os/exec"
 	"regexp"
+	"strconv"
 	"strings"
-
-	"github.com/expanse-org/go-expanse/common"
-	"github.com/expanse-org/go-expanse/crypto"
 )
 
-var (
+var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
-	versionRegexp = regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`)
-	solcParams    = []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
-		"--add-std",  // include standard lib contracts
-		"--optimize", // code optimizer switched on
-	}
-)
 
 type Contract struct {
 	Code string       `json:"code"`
@@ -54,17 +45,33 @@ type ContractInfo struct {
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
+	Metadata        string      `json:"metadata"`
 }
 
 // Solidity contains information about the solidity compiler.
 type Solidity struct {
 	Path, Version, FullVersion string
+	Major, Minor, Patch        int
 }
 
 // --combined-output format
 type solcOutput struct {
-	Contracts map[string]struct{ Bin, Abi, Devdoc, Userdoc string }
+	Contracts map[string]struct {
-	Version   string
+		Bin, Abi, Devdoc, Userdoc, Metadata string
+	}
+	Version string
+}
+
+func (s *Solidity) makeArgs() []string {
+	p := []string{
+		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--add-std",  // include standard lib contracts
+		"--optimize", // code optimizer switched on
+	}
+	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
+		p[1] += ",metadata"
+	}
+	return p
 }
 
 // SolidityVersion runs solc and parses its version output.
@@ -75,13 +82,23 @@ func SolidityVersion(solc string) (*Solidity, error) {
 	var out bytes.Buffer
 	cmd := exec.Command(solc, "--version")
 	cmd.Stdout = &out
-	if err := cmd.Run(); err != nil {
+	err := cmd.Run()
+	if err != nil {
 		return nil, err
 	}
-	s := &Solidity{
+	matches := versionRegexp.FindStringSubmatch(out.String())
-		Path:        cmd.Path,
+	if len(matches) != 4 {
-		FullVersion: out.String(),
+		return nil, fmt.Errorf("can't parse solc version %q", out.String())
-		Version:     versionRegexp.FindString(out.String()),
+	}
+	s := &Solidity{Path: cmd.Path, FullVersion: out.String(), Version: matches[0]}
+	if s.Major, err = strconv.Atoi(matches[1]); err != nil {
+		return nil, err
+	}
+	if s.Minor, err = strconv.Atoi(matches[2]); err != nil {
+		return nil, err
+	}
+	if s.Patch, err = strconv.Atoi(matches[3]); err != nil {
+		return nil, err
 	}
 	return s, nil
 }
@@ -91,13 +108,14 @@ func CompileSolidityString(solc, source string) (map[string]*Contract, error) {
 	if len(source) == 0 {
 		return nil, errors.New("solc: empty source string")
 	}
-	if solc == "" {
+	s, err := SolidityVersion(solc)
-		solc = "solc"
+	if err != nil {
+		return nil, err
 	}
-	args := append(solcParams, "--")
+	args := append(s.makeArgs(), "--")
-	cmd := exec.Command(solc, append(args, "-")...)
+	cmd := exec.Command(s.Path, append(args, "-")...)
 	cmd.Stdin = strings.NewReader(source)
-	return runsolc(cmd, source)
+	return s.run(cmd, source)
 }
 
 // CompileSolidity compiles all given Solidity source files.
@@ -109,15 +127,16 @@ func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract,
 	if err != nil {
 		return nil, err
 	}
-	if solc == "" {
+	s, err := SolidityVersion(solc)
-		solc = "solc"
+	if err != nil {
+		return nil, err
 	}
-	args := append(solcParams, "--")
+	args := append(s.makeArgs(), "--")
-	cmd := exec.Command(solc, append(args, sourcefiles...)...)
+	cmd := exec.Command(s.Path, append(args, sourcefiles...)...)
-	return runsolc(cmd, source)
+	return s.run(cmd, source)
 }
 
-func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
+func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 	var stderr, stdout bytes.Buffer
 	cmd.Stderr = &stderr
 	cmd.Stdout = &stdout
@@ -128,7 +147,6 @@ func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 	if err := json.Unmarshal(stdout.Bytes(), &output); err != nil {
 		return nil, err
 	}
-	shortVersion := versionRegexp.FindString(output.Version)
 
 	// Compilation succeeded, assemble and return the contracts.
 	contracts := make(map[string]*Contract)
@@ -151,12 +169,13 @@ func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
-				LanguageVersion: shortVersion,
+				LanguageVersion: s.Version,
-				CompilerVersion: shortVersion,
+				CompilerVersion: s.Version,
-				CompilerOptions: strings.Join(solcParams, " "),
+				CompilerOptions: strings.Join(s.makeArgs(), " "),
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,
+				Metadata:        info.Metadata,
 			},
 		}
 	}
@@ -174,13 +193,3 @@ func slurpFiles(files []string) (string, error) {
 	}
 	return concat.String(), nil
 }
-
-// SaveInfo serializes info to the given file and returns its Keccak256 hash.
-func SaveInfo(info *ContractInfo, filename string) (common.Hash, error) {
-	infojson, err := json.Marshal(info)
-	if err != nil {
-		return common.Hash{}, err
-	}
-	contenthash := common.BytesToHash(crypto.Keccak256(infojson))
-	return contenthash, ioutil.WriteFile(filename, infojson, 0600)
-}

common/compiler/solidity_test.go

@@ -17,14 +17,8 @@
 package compiler
 
 import (
-	"encoding/json"
-	"io/ioutil"
-	"os"
 	"os/exec"
-	"path"
 	"testing"
-
-	"github.com/expanse-org/go-expanse/common"
 )
 
 const (
@@ -36,7 +30,6 @@ contract test {
    }
 }
 `
-	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
 )
 
 func skipWithoutSolc(t *testing.T) {
@@ -57,7 +50,10 @@ func TestCompiler(t *testing.T) {
 	}
 	c, ok := contracts["test"]
 	if !ok {
-		t.Fatal("info for contract 'test' not present in result")
+		c, ok = contracts["<stdin>:test"]
+		if !ok {
+			t.Fatal("info for contract 'test' not present in result")
+		}
 	}
 	if c.Code == "" {
 		t.Error("empty code")
@@ -79,28 +75,3 @@ func TestCompileError(t *testing.T) {
 	}
 	t.Logf("error: %v", err)
 }
-
-func TestSaveInfo(t *testing.T) {
-	var cinfo ContractInfo
-	err := json.Unmarshal([]byte(testInfo), &cinfo)
-	if err != nil {
-		t.Errorf("%v", err)
-	}
-	filename := path.Join(os.TempDir(), "solctest.info.json")
-	os.Remove(filename)
-	cinfohash, err := SaveInfo(&cinfo, filename)
-	if err != nil {
-		t.Errorf("error extracting info: %v", err)
-	}
-	got, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Errorf("error reading '%v': %v", filename, err)
-	}
-	if string(got) != testInfo {
-		t.Errorf("incorrect info.json extracted, expected:\n%s\ngot\n%s", testInfo, string(got))
-	}
-	wantHash := common.HexToHash("0x22450a77f0c3ff7a395948d07bc1456881226a1b6325f4189cb5f1254a824080")
-	if cinfohash != wantHash {
-		t.Errorf("content hash for info is incorrect. expected %v, got %v", wantHash.Hex(), cinfohash.Hex())
-	}
-}

common/math/big.go

@@ -27,6 +27,8 @@ var (
 	tt256     = BigPow(2, 256)
 	tt256m1   = new(big.Int).Sub(tt256, big.NewInt(1))
 	MaxBig256 = new(big.Int).Set(tt256m1)
+	tt63      = BigPow(2, 63)
+	MaxBig63  = new(big.Int).Sub(tt63, big.NewInt(1))
 )
 
 const (
@@ -51,6 +53,9 @@ func (i *HexOrDecimal256) UnmarshalText(input []byte) error {
 
 // MarshalText implements encoding.TextMarshaler.
 func (i *HexOrDecimal256) MarshalText() ([]byte, error) {
+	if i == nil {
+		return []byte("0x0"), nil
+	}
 	return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil
 }
 

common/math/big_test.go

@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

consensus/clique/api.go

@@ -0,0 +1,119 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/rpc"
+)
+
+// API is a user facing RPC API to allow controlling the signer and voting
+// mechanisms of the proof-of-authority scheme.
+type API struct {
+	chain  consensus.ChainReader
+	clique *Clique
+}
+
+// GetSnapshot retrieves the state snapshot at a given block.
+func (api *API) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) {
+	// Retrieve the requested block number (or current if none requested)
+	var header *types.Header
+	if number == nil || *number == rpc.LatestBlockNumber {
+		header = api.chain.CurrentHeader()
+	} else {
+		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
+	}
+	// Ensure we have an actually valid block and return its snapshot
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+}
+
+// GetSnapshotAtHash retrieves the state snapshot at a given block.
+func (api *API) GetSnapshotAtHash(hash common.Hash) (*Snapshot, error) {
+	header := api.chain.GetHeaderByHash(hash)
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+}
+
+// GetSigners retrieves the list of authorized signers at the specified block.
+func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
+	// Retrieve the requested block number (or current if none requested)
+	var header *types.Header
+	if number == nil || *number == rpc.LatestBlockNumber {
+		header = api.chain.CurrentHeader()
+	} else {
+		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
+	}
+	// Ensure we have an actually valid block and return the signers from its snapshot
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return snap.signers(), nil
+}
+
+// GetSignersAtHash retrieves the state snapshot at a given block.
+func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
+	header := api.chain.GetHeaderByHash(hash)
+	if header == nil {
+		return nil, errUnknownBlock
+	}
+	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return snap.signers(), nil
+}
+
+// Proposals returns the current proposals the node tries to uphold and vote on.
+func (api *API) Proposals() map[common.Address]bool {
+	api.clique.lock.RLock()
+	defer api.clique.lock.RUnlock()
+
+	proposals := make(map[common.Address]bool)
+	for address, auth := range api.clique.proposals {
+		proposals[address] = auth
+	}
+	return proposals
+}
+
+// Propose injects a new authorization proposal that the signer will attempt to
+// push through.
+func (api *API) Propose(address common.Address, auth bool) {
+	api.clique.lock.Lock()
+	defer api.clique.lock.Unlock()
+
+	api.clique.proposals[address] = auth
+}
+
+// Discard drops a currently running proposal, stopping the signer from casting
+// further votes (either for or against).
+func (api *API) Discard(address common.Address) {
+	api.clique.lock.Lock()
+	defer api.clique.lock.Unlock()
+
+	delete(api.clique.proposals, address)
+}

consensus/clique/clique.go

@@ -0,0 +1,650 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package clique implements the proof-of-authority consensus engine.
+package clique
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/expanse-org/go-expanse/accounts"
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/common/hexutil"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/crypto"
+	"github.com/expanse-org/go-expanse/crypto/sha3"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/expanse-org/go-expanse/rlp"
+	"github.com/expanse-org/go-expanse/rpc"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+const (
+	checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database
+	inmemorySnapshots  = 128  // Number of recent vote snapshots to keep in memory
+	inmemorySignatures = 4096 // Number of recent block signatures to keep in memory
+
+	wiggleTime = 500 * time.Millisecond // Random delay (per signer) to allow concurrent signers
+)
+
+// Clique proof-of-authority protocol constants.
+var (
+	epochLength = uint64(30000) // Default number of blocks after which to checkpoint and reset the pending votes
+	blockPeriod = uint64(15)    // Default minimum difference between two consecutive block's timestamps
+
+	extraVanity = 32 // Fixed number of extra-data prefix bytes reserved for signer vanity
+	extraSeal   = 65 // Fixed number of extra-data suffix bytes reserved for signer seal
+
+	nonceAuthVote = hexutil.MustDecode("0xffffffffffffffff") // Magic nonce number to vote on adding a new signer
+	nonceDropVote = hexutil.MustDecode("0x0000000000000000") // Magic nonce number to vote on removing a signer.
+
+	uncleHash = types.CalcUncleHash(nil) // Always Keccak256(RLP([])) as uncles are meaningless outside of PoW.
+
+	diffInTurn = big.NewInt(2) // Block difficulty for in-turn signatures
+	diffNoTurn = big.NewInt(1) // Block difficulty for out-of-turn signatures
+)
+
+// Various error messages to mark blocks invalid. These should be private to
+// prevent engine specific errors from being referenced in the remainder of the
+// codebase, inherently breaking if the engine is swapped out. Please put common
+// error types into the consensus package.
+var (
+	// errUnknownBlock is returned when the list of signers is requested for a block
+	// that is not part of the local blockchain.
+	errUnknownBlock = errors.New("unknown block")
+
+	// errInvalidCheckpointBeneficiary is returned if a checkpoint/epoch transition
+	// block has a beneficiary set to non zeroes.
+	errInvalidCheckpointBeneficiary = errors.New("beneficiary in checkpoint block non-zero")
+
+	// errInvalidVote is returned if a nonce value is something else that the two
+	// allowed constants of 0x00..0 or 0xff..f.
+	errInvalidVote = errors.New("vote nonce not 0x00..0 or 0xff..f")
+
+	// errInvalidCheckpointVote is returned if a checkpoint/epoch transition block
+	// has a vote nonce set to non zeroes.
+	errInvalidCheckpointVote = errors.New("vote nonce in checkpoint block non-zero")
+
+	// errMissingVanity is returned if a block's extra-data section is shorter than
+	// 32 bytes, which is required to store the signer vanity.
+	errMissingVanity = errors.New("extra-data 32 byte vanity prefix missing")
+
+	// errMissingSignature is returned if a block's extra-data section doesn't seem
+	// to contain a 65 byte secp256k1 signature.
+	errMissingSignature = errors.New("extra-data 65 byte suffix signature missing")
+
+	// errExtraSigners is returned if non-checkpoint block contain signer data in
+	// their extra-data fields.
+	errExtraSigners = errors.New("non-checkpoint block contains extra signer list")
+
+	// drrInvalidCheckpointSigners is returned if a checkpoint block contains an
+	// invalid list of signers (i.e. non divisible by 20 bytes, or not the correct
+	// ones).
+	drrInvalidCheckpointSigners = errors.New("invalid signer list on checkpoint block")
+
+	// errInvalidMixDigest is returned if a block's mix digest is non zero.
+	errInvalidMixDigest = errors.New("non-zero mix digest")
+
+	// errInvalidUncleHash is returned if a block contains an non-empty uncle list.
+	errInvalidUncleHash = errors.New("non empty uncle hash")
+
+	// errInvalidDifficulty is returned if the difficulty of a block is not either
+	// of 1 or 2, or if the value does not match the turn of the signer.
+	errInvalidDifficulty = errors.New("invalid difficulty")
+
+	// ErrInvalidTimestamp is returned if the timestamp of a block is lower than
+	// the previous block's timestamp + the minimum block period.
+	ErrInvalidTimestamp = errors.New("invalid timestamp")
+
+	// errInvalidVotingChain is returned if an authorization list is attempted to
+	// be modified via out-of-range or non-contiguous headers.
+	errInvalidVotingChain = errors.New("invalid voting chain")
+
+	// errUnauthorized is returned if a header is signed by a non authorized entity.
+	errUnauthorized = errors.New("unauthorized")
+)
+
+// SignerFn is a signer callback function to request a hash to be signed by a
+// backing account.
+type SignerFn func(accounts.Account, []byte) ([]byte, error)
+
+// sigHash returns the hash which is used as input for the proof-of-authority
+// signing. It is the hash of the entire header apart from the 65 byte signature
+// contained at the end of the extra data.
+//
+// Note, the method requires the extra data to be at least 65 bytes, otherwise it
+// panics. This is done to avoid accidentally using both forms (signature present
+// or not), which could be abused to produce different hashes for the same header.
+func sigHash(header *types.Header) (hash common.Hash) {
+	hasher := sha3.NewKeccak256()
+
+	rlp.Encode(hasher, []interface{}{
+		header.ParentHash,
+		header.UncleHash,
+		header.Coinbase,
+		header.Root,
+		header.TxHash,
+		header.ReceiptHash,
+		header.Bloom,
+		header.Difficulty,
+		header.Number,
+		header.GasLimit,
+		header.GasUsed,
+		header.Time,
+		header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
+		header.MixDigest,
+		header.Nonce,
+	})
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+// ecrecover extracts the Ethereum account address from a signed header.
+func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, error) {
+	// If the signature's already cached, return that
+	hash := header.Hash()
+	if address, known := sigcache.Get(hash); known {
+		return address.(common.Address), nil
+	}
+	// Retrieve the signature from the header extra-data
+	if len(header.Extra) < extraSeal {
+		return common.Address{}, errMissingSignature
+	}
+	signature := header.Extra[len(header.Extra)-extraSeal:]
+
+	// Recover the public key and the Ethereum address
+	pubkey, err := crypto.Ecrecover(sigHash(header).Bytes(), signature)
+	if err != nil {
+		return common.Address{}, err
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+
+	sigcache.Add(hash, signer)
+	return signer, nil
+}
+
+// Clique is the proof-of-authority consensus engine proposed to support the
+// Ethereum testnet following the Ropsten attacks.
+type Clique struct {
+	config *params.CliqueConfig // Consensus engine configuration parameters
+	db     ethdb.Database       // Database to store and retrieve snapshot checkpoints
+
+	recents    *lru.ARCCache // Snapshots for recent block to speed up reorgs
+	signatures *lru.ARCCache // Signatures of recent blocks to speed up mining
+
+	proposals map[common.Address]bool // Current list of proposals we are pushing
+
+	signer common.Address // Ethereum address of the signing key
+	signFn SignerFn       // Signer function to authorize hashes with
+	lock   sync.RWMutex   // Protects the signer fields
+}
+
+// New creates a Clique proof-of-authority consensus engine with the initial
+// signers set to the ones provided by the user.
+func New(config *params.CliqueConfig, db ethdb.Database) *Clique {
+	// Set any missing consensus parameters to their defaults
+	conf := *config
+	if conf.Epoch == 0 {
+		conf.Epoch = epochLength
+	}
+	if conf.Period == 0 {
+		conf.Period = blockPeriod
+	}
+	// Allocate the snapshot caches and create the engine
+	recents, _ := lru.NewARC(inmemorySnapshots)
+	signatures, _ := lru.NewARC(inmemorySignatures)
+
+	return &Clique{
+		config:     &conf,
+		db:         db,
+		recents:    recents,
+		signatures: signatures,
+		proposals:  make(map[common.Address]bool),
+	}
+}
+
+// Author implements consensus.Engine, returning the Ethereum address recovered
+// from the signature in the header's extra-data section.
+func (c *Clique) Author(header *types.Header) (common.Address, error) {
+	return ecrecover(header, c.signatures)
+}
+
+// VerifyHeader checks whether a header conforms to the consensus rules.
+func (c *Clique) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error {
+	return c.verifyHeader(chain, header, nil)
+}
+
+// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers. The
+// method returns a quit channel to abort the operations and a results channel to
+// retrieve the async verifications (the order is that of the input slice).
+func (c *Clique) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
+	abort := make(chan struct{})
+	results := make(chan error, len(headers))
+
+	go func() {
+		for i, header := range headers {
+			err := c.verifyHeader(chain, header, headers[:i])
+
+			select {
+			case <-abort:
+				return
+			case results <- err:
+			}
+		}
+	}()
+	return abort, results
+}
+
+// verifyHeader checks whether a header conforms to the consensus rules.The
+// caller may optionally pass in a batch of parents (ascending order) to avoid
+// looking those up from the database. This is useful for concurrently verifying
+// a batch of new headers.
+func (c *Clique) verifyHeader(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	if header.Number == nil {
+		return errUnknownBlock
+	}
+	number := header.Number.Uint64()
+
+	// Don't waste time checking blocks from the future
+	if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+		return consensus.ErrFutureBlock
+	}
+	// Checkpoint blocks need to enforce zero beneficiary
+	checkpoint := (number % c.config.Epoch) == 0
+	if checkpoint && header.Coinbase != (common.Address{}) {
+		return errInvalidCheckpointBeneficiary
+	}
+	// Nonces must be 0x00..0 or 0xff..f, zeroes enforced on checkpoints
+	if !bytes.Equal(header.Nonce[:], nonceAuthVote) && !bytes.Equal(header.Nonce[:], nonceDropVote) {
+		return errInvalidVote
+	}
+	if checkpoint && !bytes.Equal(header.Nonce[:], nonceDropVote) {
+		return errInvalidCheckpointVote
+	}
+	// Check that the extra-data contains both the vanity and signature
+	if len(header.Extra) < extraVanity {
+		return errMissingVanity
+	}
+	if len(header.Extra) < extraVanity+extraSeal {
+		return errMissingSignature
+	}
+	// Ensure that the extra-data contains a signer list on checkpoint, but none otherwise
+	signersBytes := len(header.Extra) - extraVanity - extraSeal
+	if !checkpoint && signersBytes != 0 {
+		return errExtraSigners
+	}
+	if checkpoint && signersBytes%common.AddressLength != 0 {
+		return drrInvalidCheckpointSigners
+	}
+	// Ensure that the mix digest is zero as we don't have fork protection currently
+	if header.MixDigest != (common.Hash{}) {
+		return errInvalidMixDigest
+	}
+	// Ensure that the block doesn't contain any uncles which are meaningless in PoA
+	if header.UncleHash != uncleHash {
+		return errInvalidUncleHash
+	}
+	// Ensure that the block's difficulty is meaningful (may not be correct at this point)
+	if number > 0 {
+		if header.Difficulty == nil || (header.Difficulty.Cmp(diffInTurn) != 0 && header.Difficulty.Cmp(diffNoTurn) != 0) {
+			return errInvalidDifficulty
+		}
+	}
+	// All basic checks passed, verify cascading fields
+	return c.verifyCascadingFields(chain, header, parents)
+}
+
+// verifyCascadingFields verifies all the header fields that are not standalone,
+// rather depend on a batch of previous headers. The caller may optionally pass
+// in a batch of parents (ascending order) to avoid looking those up from the
+// database. This is useful for concurrently verifying a batch of new headers.
+func (c *Clique) verifyCascadingFields(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	// The genesis block is the always valid dead-end
+	number := header.Number.Uint64()
+	if number == 0 {
+		return nil
+	}
+	// Ensure that the block's timestamp isn't too close to it's parent
+	var parent *types.Header
+	if len(parents) > 0 {
+		parent = parents[len(parents)-1]
+	} else {
+		parent = chain.GetHeader(header.ParentHash, number-1)
+	}
+	if parent == nil || parent.Number.Uint64() != number-1 || parent.Hash() != header.ParentHash {
+		return consensus.ErrUnknownAncestor
+	}
+	if parent.Time.Uint64()+c.config.Period > header.Time.Uint64() {
+		return ErrInvalidTimestamp
+	}
+	// Retrieve the snapshot needed to verify this header and cache it
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, parents)
+	if err != nil {
+		return err
+	}
+	// If the block is a checkpoint block, verify the signer list
+	if number%c.config.Epoch == 0 {
+		signers := make([]byte, len(snap.Signers)*common.AddressLength)
+		for i, signer := range snap.signers() {
+			copy(signers[i*common.AddressLength:], signer[:])
+		}
+		extraSuffix := len(header.Extra) - extraSeal
+		if !bytes.Equal(header.Extra[extraVanity:extraSuffix], signers) {
+			return drrInvalidCheckpointSigners
+		}
+	}
+	// All basic checks passed, verify the seal and return
+	return c.verifySeal(chain, header, parents)
+}
+
+// snapshot retrieves the authorization snapshot at a given point in time.
+func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash common.Hash, parents []*types.Header) (*Snapshot, error) {
+	// Search for a snapshot in memory or on disk for checkpoints
+	var (
+		headers []*types.Header
+		snap    *Snapshot
+	)
+	for snap == nil {
+		// If an in-memory snapshot was found, use that
+		if s, ok := c.recents.Get(hash); ok {
+			snap = s.(*Snapshot)
+			break
+		}
+		// If an on-disk checkpoint snapshot can be found, use that
+		if number%checkpointInterval == 0 {
+			if s, err := loadSnapshot(c.config, c.signatures, c.db, hash); err == nil {
+				log.Trace("Loaded voting snapshot form disk", "number", number, "hash", hash)
+				snap = s
+				break
+			}
+		}
+		// If we're at block zero, make a snapshot
+		if number == 0 {
+			genesis := chain.GetHeaderByNumber(0)
+			if err := c.VerifyHeader(chain, genesis, false); err != nil {
+				return nil, err
+			}
+			signers := make([]common.Address, (len(genesis.Extra)-extraVanity-extraSeal)/common.AddressLength)
+			for i := 0; i < len(signers); i++ {
+				copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:])
+			}
+			snap = newSnapshot(c.config, c.signatures, 0, genesis.Hash(), signers)
+			if err := snap.store(c.db); err != nil {
+				return nil, err
+			}
+			log.Trace("Stored genesis voting snapshot to disk")
+			break
+		}
+		// No snapshot for this header, gather the header and move backward
+		var header *types.Header
+		if len(parents) > 0 {
+			// If we have explicit parents, pick from there (enforced)
+			header = parents[len(parents)-1]
+			if header.Hash() != hash || header.Number.Uint64() != number {
+				return nil, consensus.ErrUnknownAncestor
+			}
+			parents = parents[:len(parents)-1]
+		} else {
+			// No explicit parents (or no more left), reach out to the database
+			header = chain.GetHeader(hash, number)
+			if header == nil {
+				return nil, consensus.ErrUnknownAncestor
+			}
+		}
+		headers = append(headers, header)
+		number, hash = number-1, header.ParentHash
+	}
+	// Previous snapshot found, apply any pending headers on top of it
+	for i := 0; i < len(headers)/2; i++ {
+		headers[i], headers[len(headers)-1-i] = headers[len(headers)-1-i], headers[i]
+	}
+	snap, err := snap.apply(headers)
+	if err != nil {
+		return nil, err
+	}
+	c.recents.Add(snap.Hash, snap)
+
+	// If we've generated a new checkpoint snapshot, save to disk
+	if snap.Number%checkpointInterval == 0 && len(headers) > 0 {
+		if err = snap.store(c.db); err != nil {
+			return nil, err
+		}
+		log.Trace("Stored voting snapshot to disk", "number", snap.Number, "hash", snap.Hash)
+	}
+	return snap, err
+}
+
+// VerifyUncles implements consensus.Engine, always returning an error for any
+// uncles as this consensus mechanism doesn't permit uncles.
+func (c *Clique) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
+	if len(block.Uncles()) > 0 {
+		return errors.New("uncles not allowed")
+	}
+	return nil
+}
+
+// VerifySeal implements consensus.Engine, checking whether the signature contained
+// in the header satisfies the consensus protocol requirements.
+func (c *Clique) VerifySeal(chain consensus.ChainReader, header *types.Header) error {
+	return c.verifySeal(chain, header, nil)
+}
+
+// verifySeal checks whether the signature contained in the header satisfies the
+// consensus protocol requirements. The method accepts an optional list of parent
+// headers that aren't yet part of the local blockchain to generate the snapshots
+// from.
+func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, parents []*types.Header) error {
+	// Verifying the genesis block is not supported
+	number := header.Number.Uint64()
+	if number == 0 {
+		return errUnknownBlock
+	}
+	// Retrieve the snapshot needed to verify this header and cache it
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, parents)
+	if err != nil {
+		return err
+	}
+	c.recents.Add(snap.Hash, snap)
+
+	// Resolve the authorization key and check against signers
+	signer, err := ecrecover(header, c.signatures)
+	if err != nil {
+		return err
+	}
+	if _, ok := snap.Signers[signer]; !ok {
+		return errUnauthorized
+	}
+	for seen, recent := range snap.Recents {
+		if recent == signer {
+			// Signer is among recents, only fail if the current block doens't shift it out
+			if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit {
+				return errUnauthorized
+			}
+		}
+	}
+	// Ensure that the difficulty corresponts to the turn-ness of the signer
+	inturn := snap.inturn(header.Number.Uint64(), signer)
+	if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
+		return errInvalidDifficulty
+	}
+	if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
+		return errInvalidDifficulty
+	}
+	return nil
+}
+
+// Prepare implements consensus.Engine, preparing all the consensus fields of the
+// header for running the transactions on top.
+func (c *Clique) Prepare(chain consensus.ChainReader, header *types.Header) error {
+	// If the block isn't a checkpoint, cast a random vote (good enough fror now)
+	header.Coinbase = common.Address{}
+	header.Nonce = types.BlockNonce{}
+
+	number := header.Number.Uint64()
+	if number%c.config.Epoch != 0 {
+		c.lock.RLock()
+		if len(c.proposals) > 0 {
+			addresses := make([]common.Address, 0, len(c.proposals))
+			for address := range c.proposals {
+				addresses = append(addresses, address)
+			}
+			header.Coinbase = addresses[rand.Intn(len(addresses))]
+			if c.proposals[header.Coinbase] {
+				copy(header.Nonce[:], nonceAuthVote)
+			} else {
+				copy(header.Nonce[:], nonceDropVote)
+			}
+		}
+		c.lock.RUnlock()
+	}
+	// Assemble the voting snapshot and set the correct difficulty
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
+	if err != nil {
+		return err
+	}
+	header.Difficulty = diffNoTurn
+	if snap.inturn(header.Number.Uint64(), c.signer) {
+		header.Difficulty = diffInTurn
+	}
+	// Ensure the extra data has all it's components
+	if len(header.Extra) < extraVanity {
+		header.Extra = append(header.Extra, bytes.Repeat([]byte{0x00}, extraVanity-len(header.Extra))...)
+	}
+	header.Extra = header.Extra[:extraVanity]
+
+	if number%c.config.Epoch == 0 {
+		for _, signer := range snap.signers() {
+			header.Extra = append(header.Extra, signer[:]...)
+		}
+	}
+	header.Extra = append(header.Extra, make([]byte, extraSeal)...)
+
+	// Mix digest is reserved for now, set to empty
+	header.MixDigest = common.Hash{}
+
+	// Ensure the timestamp has the correct delay
+	parent := chain.GetHeader(header.ParentHash, number-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	header.Time = new(big.Int).Add(parent.Time, new(big.Int).SetUint64(c.config.Period))
+	if header.Time.Int64() < time.Now().Unix() {
+		header.Time = big.NewInt(time.Now().Unix())
+	}
+	return nil
+}
+
+// Finalize implements consensus.Engine, ensuring no uncles are set, nor block
+// rewards given, and returns the final block.
+func (c *Clique) Finalize(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
+	// No block rewards in PoA, so the state remains as is and uncles are dropped
+	header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
+	header.UncleHash = types.CalcUncleHash(nil)
+
+	// Assemble and return the final block for sealing
+	return types.NewBlock(header, txs, nil, receipts), nil
+}
+
+// Authorize injects a private key into the consensus engine to mint new blocks
+// with.
+func (c *Clique) Authorize(signer common.Address, signFn SignerFn) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.signer = signer
+	c.signFn = signFn
+}
+
+// Seal implements consensus.Engine, attempting to create a sealed block using
+// the local signing credentials.
+func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+	header := block.Header()
+
+	// Sealing the genesis block is not supported
+	number := header.Number.Uint64()
+	if number == 0 {
+		return nil, errUnknownBlock
+	}
+	// Don't hold the signer fields for the entire sealing procedure
+	c.lock.RLock()
+	signer, signFn := c.signer, c.signFn
+	c.lock.RUnlock()
+
+	// Bail out if we're unauthorized to sign a block
+	snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
+	if err != nil {
+		return nil, err
+	}
+	if _, authorized := snap.Signers[signer]; !authorized {
+		return nil, errUnauthorized
+	}
+	// If we're amongs the recent signers, wait for the next block
+	for seen, recent := range snap.Recents {
+		if recent == signer {
+			// Signer is among recents, only wait if the current block doens't shift it out
+			if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit {
+				log.Info("Signed recently, must wait for others")
+				<-stop
+				return nil, nil
+			}
+		}
+	}
+	// Sweet, the protocol permits us to sign the block, wait for our time
+	delay := time.Unix(header.Time.Int64(), 0).Sub(time.Now())
+	if header.Difficulty.Cmp(diffNoTurn) == 0 {
+		// It's not our turn explicitly to sign, delay it a bit
+		wiggle := time.Duration(len(snap.Signers)/2+1) * wiggleTime
+		delay += time.Duration(rand.Int63n(int64(wiggle)))
+
+		log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
+	}
+	log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
+
+	select {
+	case <-stop:
+		return nil, nil
+	case <-time.After(delay):
+	}
+	// Sign all the things!
+	sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
+	if err != nil {
+		return nil, err
+	}
+	copy(header.Extra[len(header.Extra)-extraSeal:], sighash)
+
+	return block.WithSeal(header), nil
+}
+
+// APIs implements consensus.Engine, returning the user facing RPC API to allow
+// controlling the signer voting.
+func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API {
+	return []rpc.API{{
+		Namespace: "clique",
+		Version:   "1.0",
+		Service:   &API{chain: chain, clique: c},
+		Public:    false,
+	}}
+}

consensus/clique/snapshot.go

@@ -0,0 +1,304 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/params"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+// Vote represents a single vote that an authorized signer made to modify the
+// list of authorizations.
+type Vote struct {
+	Signer    common.Address `json:"signer"`    // Authorized signer that cast this vote
+	Block     uint64         `json:"block"`     // Block number the vote was cast in (expire old votes)
+	Address   common.Address `json:"address"`   // Account being voted on to change its authorization
+	Authorize bool           `json:"authorize"` // Whether to authorize or deauthorize the voted account
+}
+
+// Tally is a simple vote tally to keep the current score of votes. Votes that
+// go against the proposal aren't counted since it's equivalent to not voting.
+type Tally struct {
+	Authorize bool `json:"authorize"` // Whether the vote it about authorizing or kicking someone
+	Votes     int  `json:"votes"`     // Number of votes until now wanting to pass the proposal
+}
+
+// Snapshot is the state of the authorization voting at a given point in time.
+type Snapshot struct {
+	config   *params.CliqueConfig // Consensus engine parameters to fine tune behavior
+	sigcache *lru.ARCCache        // Cache of recent block signatures to speed up ecrecover
+
+	Number  uint64                      `json:"number"`  // Block number where the snapshot was created
+	Hash    common.Hash                 `json:"hash"`    // Block hash where the snapshot was created
+	Signers map[common.Address]struct{} `json:"signers"` // Set of authorized signers at this moment
+	Recents map[uint64]common.Address   `json:"recents"` // Set of recent signers for spam protections
+	Votes   []*Vote                     `json:"votes"`   // List of votes cast in chronological order
+	Tally   map[common.Address]Tally    `json:"tally"`   // Current vote tally to avoid recalculating
+}
+
+// newSnapshot create a new snapshot with the specified startup parameters. This
+// method does not initialize the set of recent signers, so only ever use if for
+// the genesis block.
+func newSnapshot(config *params.CliqueConfig, sigcache *lru.ARCCache, number uint64, hash common.Hash, signers []common.Address) *Snapshot {
+	snap := &Snapshot{
+		config:   config,
+		sigcache: sigcache,
+		Number:   number,
+		Hash:     hash,
+		Signers:  make(map[common.Address]struct{}),
+		Recents:  make(map[uint64]common.Address),
+		Tally:    make(map[common.Address]Tally),
+	}
+	for _, signer := range signers {
+		snap.Signers[signer] = struct{}{}
+	}
+	return snap
+}
+
+// loadSnapshot loads an existing snapshot from the database.
+func loadSnapshot(config *params.CliqueConfig, sigcache *lru.ARCCache, db ethdb.Database, hash common.Hash) (*Snapshot, error) {
+	blob, err := db.Get(append([]byte("clique-"), hash[:]...))
+	if err != nil {
+		return nil, err
+	}
+	snap := new(Snapshot)
+	if err := json.Unmarshal(blob, snap); err != nil {
+		return nil, err
+	}
+	snap.config = config
+	snap.sigcache = sigcache
+
+	return snap, nil
+}
+
+// store inserts the snapshot into the database.
+func (s *Snapshot) store(db ethdb.Database) error {
+	blob, err := json.Marshal(s)
+	if err != nil {
+		return err
+	}
+	return db.Put(append([]byte("clique-"), s.Hash[:]...), blob)
+}
+
+// copy creates a deep copy of the snapshot, though not the individual votes.
+func (s *Snapshot) copy() *Snapshot {
+	cpy := &Snapshot{
+		config:   s.config,
+		sigcache: s.sigcache,
+		Number:   s.Number,
+		Hash:     s.Hash,
+		Signers:  make(map[common.Address]struct{}),
+		Recents:  make(map[uint64]common.Address),
+		Votes:    make([]*Vote, len(s.Votes)),
+		Tally:    make(map[common.Address]Tally),
+	}
+	for signer := range s.Signers {
+		cpy.Signers[signer] = struct{}{}
+	}
+	for block, signer := range s.Recents {
+		cpy.Recents[block] = signer
+	}
+	for address, tally := range s.Tally {
+		cpy.Tally[address] = tally
+	}
+	copy(cpy.Votes, s.Votes)
+
+	return cpy
+}
+
+// cast adds a new vote into the tally.
+func (s *Snapshot) cast(address common.Address, authorize bool) bool {
+	// Ensure the vote is meaningful
+	_, signer := s.Signers[address]
+	if (signer && authorize) || (!signer && !authorize) {
+		return false
+	}
+	// Cast the vote into an existing or new tally
+	if old, ok := s.Tally[address]; ok {
+		old.Votes++
+		s.Tally[address] = old
+	} else {
+		s.Tally[address] = Tally{Authorize: authorize, Votes: 1}
+	}
+	return true
+}
+
+// uncast removes a previously cast vote from the tally.
+func (s *Snapshot) uncast(address common.Address, authorize bool) bool {
+	// If there's no tally, it's a dangling vote, just drop
+	tally, ok := s.Tally[address]
+	if !ok {
+		return false
+	}
+	// Ensure we only revert counted votes
+	if tally.Authorize != authorize {
+		return false
+	}
+	// Otherwise revert the vote
+	if tally.Votes > 1 {
+		tally.Votes--
+		s.Tally[address] = tally
+	} else {
+		delete(s.Tally, address)
+	}
+	return true
+}
+
+// apply creates a new authorization snapshot by applying the given headers to
+// the original one.
+func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) {
+	// Allow passing in no headers for cleaner code
+	if len(headers) == 0 {
+		return s, nil
+	}
+	// Sanity check that the headers can be applied
+	for i := 0; i < len(headers)-1; i++ {
+		if headers[i+1].Number.Uint64() != headers[i].Number.Uint64()+1 {
+			return nil, errInvalidVotingChain
+		}
+	}
+	if headers[0].Number.Uint64() != s.Number+1 {
+		return nil, errInvalidVotingChain
+	}
+	// Iterate through the headers and create a new snapshot
+	snap := s.copy()
+
+	for _, header := range headers {
+		// Remove any votes on checkpoint blocks
+		number := header.Number.Uint64()
+		if number%s.config.Epoch == 0 {
+			snap.Votes = nil
+			snap.Tally = make(map[common.Address]Tally)
+		}
+		// Delete the oldest signer from the recent list to allow it signing again
+		if limit := uint64(len(snap.Signers)/2 + 1); number >= limit {
+			delete(snap.Recents, number-limit)
+		}
+		// Resolve the authorization key and check against signers
+		signer, err := ecrecover(header, s.sigcache)
+		if err != nil {
+			return nil, err
+		}
+		if _, ok := snap.Signers[signer]; !ok {
+			return nil, errUnauthorized
+		}
+		for _, recent := range snap.Recents {
+			if recent == signer {
+				return nil, errUnauthorized
+			}
+		}
+		snap.Recents[number] = signer
+
+		// Header authorized, discard any previous votes from the signer
+		for i, vote := range snap.Votes {
+			if vote.Signer == signer && vote.Address == header.Coinbase {
+				// Uncast the vote from the cached tally
+				snap.uncast(vote.Address, vote.Authorize)
+
+				// Uncast the vote from the chronological list
+				snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+				break // only one vote allowed
+			}
+		}
+		// Tally up the new vote from the signer
+		var authorize bool
+		switch {
+		case bytes.Compare(header.Nonce[:], nonceAuthVote) == 0:
+			authorize = true
+		case bytes.Compare(header.Nonce[:], nonceDropVote) == 0:
+			authorize = false
+		default:
+			return nil, errInvalidVote
+		}
+		if snap.cast(header.Coinbase, authorize) {
+			snap.Votes = append(snap.Votes, &Vote{
+				Signer:    signer,
+				Block:     number,
+				Address:   header.Coinbase,
+				Authorize: authorize,
+			})
+		}
+		// If the vote passed, update the list of signers
+		if tally := snap.Tally[header.Coinbase]; tally.Votes > len(snap.Signers)/2 {
+			if tally.Authorize {
+				snap.Signers[header.Coinbase] = struct{}{}
+			} else {
+				delete(snap.Signers, header.Coinbase)
+
+				// Signer list shrunk, delete any leftover recent caches
+				if limit := uint64(len(snap.Signers)/2 + 1); number >= limit {
+					delete(snap.Recents, number-limit)
+				}
+				// Discard any previous votes the deauthorized signer cast
+				for i := 0; i < len(snap.Votes); i++ {
+					if snap.Votes[i].Signer == header.Coinbase {
+						// Uncast the vote from the cached tally
+						snap.uncast(snap.Votes[i].Address, snap.Votes[i].Authorize)
+
+						// Uncast the vote from the chronological list
+						snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+
+						i--
+					}
+				}
+			}
+			// Discard any previous votes around the just changed account
+			for i := 0; i < len(snap.Votes); i++ {
+				if snap.Votes[i].Address == header.Coinbase {
+					snap.Votes = append(snap.Votes[:i], snap.Votes[i+1:]...)
+					i--
+				}
+			}
+			delete(snap.Tally, header.Coinbase)
+		}
+	}
+	snap.Number += uint64(len(headers))
+	snap.Hash = headers[len(headers)-1].Hash()
+
+	return snap, nil
+}
+
+// signers retrieves the list of authorized signers in ascending order.
+func (s *Snapshot) signers() []common.Address {
+	signers := make([]common.Address, 0, len(s.Signers))
+	for signer := range s.Signers {
+		signers = append(signers, signer)
+	}
+	for i := 0; i < len(signers); i++ {
+		for j := i + 1; j < len(signers); j++ {
+			if bytes.Compare(signers[i][:], signers[j][:]) > 0 {
+				signers[i], signers[j] = signers[j], signers[i]
+			}
+		}
+	}
+	return signers
+}
+
+// inturn returns if a signer at a given block height is in-turn or not.
+func (s *Snapshot) inturn(number uint64, signer common.Address) bool {
+	signers, offset := s.signers(), 0
+	for offset < len(signers) && signers[offset] != signer {
+		offset++
+	}
+	return (number % uint64(len(signers))) == uint64(offset)
+}

consensus/clique/snapshot_test.go

@@ -0,0 +1,405 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package clique
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"math/big"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/crypto"
+	"github.com/expanse-org/go-expanse/ethdb"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+type testerVote struct {
+	signer string
+	voted  string
+	auth   bool
+}
+
+// testerAccountPool is a pool to maintain currently active tester accounts,
+// mapped from textual names used in the tests below to actual Ethereum private
+// keys capable of signing transactions.
+type testerAccountPool struct {
+	accounts map[string]*ecdsa.PrivateKey
+}
+
+func newTesterAccountPool() *testerAccountPool {
+	return &testerAccountPool{
+		accounts: make(map[string]*ecdsa.PrivateKey),
+	}
+}
+
+func (ap *testerAccountPool) sign(header *types.Header, signer string) {
+	// Ensure we have a persistent key for the signer
+	if ap.accounts[signer] == nil {
+		ap.accounts[signer], _ = crypto.GenerateKey()
+	}
+	// Sign the header and embed the signature in extra data
+	sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
+	copy(header.Extra[len(header.Extra)-65:], sig)
+}
+
+func (ap *testerAccountPool) address(account string) common.Address {
+	// Ensure we have a persistent key for the account
+	if ap.accounts[account] == nil {
+		ap.accounts[account], _ = crypto.GenerateKey()
+	}
+	// Resolve and return the Ethereum address
+	return crypto.PubkeyToAddress(ap.accounts[account].PublicKey)
+}
+
+// testerChainReader implements consensus.ChainReader to access the genesis
+// block. All other methods and requests will panic.
+type testerChainReader struct {
+	db ethdb.Database
+}
+
+func (r *testerChainReader) Config() *params.ChainConfig                 { panic("not supported") }
+func (r *testerChainReader) CurrentHeader() *types.Header                { panic("not supported") }
+func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
+func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block   { panic("not supported") }
+func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header   { panic("not supported") }
+func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header {
+	if number == 0 {
+		return core.GetHeader(r.db, core.GetCanonicalHash(r.db, 0), 0)
+	}
+	panic("not supported")
+}
+
+// Tests that voting is evaluated correctly for various simple and complex scenarios.
+func TestVoting(t *testing.T) {
+	// Define the various voting scenarios to test
+	tests := []struct {
+		epoch   uint64
+		signers []string
+		votes   []testerVote
+		results []string
+	}{
+		{
+			// Single signer, no votes cast
+			signers: []string{"A"},
+			votes:   []testerVote{{signer: "A"}},
+			results: []string{"A"},
+		}, {
+			// Single signer, voting to add two others (only accept first, second needs 2 votes)
+			signers: []string{"A"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Two signers, voting to add three others (only accept first two, third needs 3 votes already)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B", voted: "C", auth: true},
+				{signer: "A", voted: "D", auth: true},
+				{signer: "B", voted: "D", auth: true},
+				{signer: "C"},
+				{signer: "A", voted: "E", auth: true},
+				{signer: "B", voted: "E", auth: true},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Single signer, dropping itself (weird, but one less cornercase by explicitly allowing this)
+			signers: []string{"A"},
+			votes: []testerVote{
+				{signer: "A", voted: "A", auth: false},
+			},
+			results: []string{},
+		}, {
+			// Two signers, actually needing mutual consent to drop either of them (not fulfilled)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Two signers, actually needing mutual consent to drop either of them (fulfilled)
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B", voted: "B", auth: false},
+			},
+			results: []string{"A"},
+		}, {
+			// Three signers, two of them deciding to drop the third
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Four signers, consensus of two not being enough to drop anyone
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Four signers, consensus of three already being enough to drop someone
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Authorizations are counted once per signer per target
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Authorizing multiple accounts concurrently is permitted
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A", voted: "D", auth: true},
+				{signer: "B"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: true},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B", "C", "D"},
+		}, {
+			// Deauthorizations are counted once per signer per target
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B"},
+				{signer: "A", voted: "B", auth: false},
+				{signer: "B"},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Deauthorizing multiple accounts concurrently is permitted
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Votes from deauthorized signers are discarded immediately (deauth votes)
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "C", voted: "B", auth: false},
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Votes from deauthorized signers are discarded immediately (auth votes)
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "C", voted: "B", auth: false},
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "A", voted: "B", auth: false},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Cascading changes are not allowed, only the the account being voted on may change
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Changes reaching consensus out of bounds (via a deauth) execute on touch
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "C", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		}, {
+			// Changes reaching consensus out of bounds (via a deauth) may go out of consensus on first touch
+			signers: []string{"A", "B", "C", "D"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: false},
+				{signer: "B"},
+				{signer: "C"},
+				{signer: "A", voted: "D", auth: false},
+				{signer: "B", voted: "C", auth: false},
+				{signer: "C"},
+				{signer: "A"},
+				{signer: "B", voted: "D", auth: false},
+				{signer: "C", voted: "D", auth: false},
+				{signer: "A"},
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B", "C"},
+		}, {
+			// Ensure that pending votes don't survive authorization status changes. This
+			// corner case can only appear if a signer is quickly added, remove and then
+			// readded (or the inverse), while one of the original voters dropped. If a
+			// past vote is left cached in the system somewhere, this will interfere with
+			// the final signer outcome.
+			signers: []string{"A", "B", "C", "D", "E"},
+			votes: []testerVote{
+				{signer: "A", voted: "F", auth: true}, // Authorize F, 3 votes needed
+				{signer: "B", voted: "F", auth: true},
+				{signer: "C", voted: "F", auth: true},
+				{signer: "D", voted: "F", auth: false}, // Deauthorize F, 4 votes needed (leave A's previous vote "unchanged")
+				{signer: "E", voted: "F", auth: false},
+				{signer: "B", voted: "F", auth: false},
+				{signer: "C", voted: "F", auth: false},
+				{signer: "D", voted: "F", auth: true}, // Almost authorize F, 2/3 votes needed
+				{signer: "E", voted: "F", auth: true},
+				{signer: "B", voted: "A", auth: false}, // Deauthorize A, 3 votes needed
+				{signer: "C", voted: "A", auth: false},
+				{signer: "D", voted: "A", auth: false},
+				{signer: "B", voted: "F", auth: true}, // Finish authorizing F, 3/3 votes needed
+			},
+			results: []string{"B", "C", "D", "E", "F"},
+		}, {
+			// Epoch transitions reset all votes to allow chain checkpointing
+			epoch:   3,
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A", voted: "C", auth: true},
+				{signer: "B"},
+				{signer: "A"}, // Checkpoint block, (don't vote here, it's validated outside of snapshots)
+				{signer: "B", voted: "C", auth: true},
+			},
+			results: []string{"A", "B"},
+		},
+	}
+	// Run through the scenarios and test them
+	for i, tt := range tests {
+		// Create the account pool and generate the initial set of signers
+		accounts := newTesterAccountPool()
+
+		signers := make([]common.Address, len(tt.signers))
+		for j, signer := range tt.signers {
+			signers[j] = accounts.address(signer)
+		}
+		for j := 0; j < len(signers); j++ {
+			for k := j + 1; k < len(signers); k++ {
+				if bytes.Compare(signers[j][:], signers[k][:]) > 0 {
+					signers[j], signers[k] = signers[k], signers[j]
+				}
+			}
+		}
+		// Create the genesis block with the initial set of signers
+		genesis := &core.Genesis{
+			ExtraData: make([]byte, extraVanity+common.AddressLength*len(signers)+extraSeal),
+		}
+		for j, signer := range signers {
+			copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:])
+		}
+		// Create a pristine blockchain with the genesis injected
+		db, _ := ethdb.NewMemDatabase()
+		genesis.Commit(db)
+
+		// Assemble a chain of headers from the cast votes
+		headers := make([]*types.Header, len(tt.votes))
+		for j, vote := range tt.votes {
+			headers[j] = &types.Header{
+				Number:   big.NewInt(int64(j) + 1),
+				Time:     big.NewInt(int64(j) * int64(blockPeriod)),
+				Coinbase: accounts.address(vote.voted),
+				Extra:    make([]byte, extraVanity+extraSeal),
+			}
+			if j > 0 {
+				headers[j].ParentHash = headers[j-1].Hash()
+			}
+			if vote.auth {
+				copy(headers[j].Nonce[:], nonceAuthVote)
+			}
+			accounts.sign(headers[j], vote.signer)
+		}
+		// Pass all the headers through clique and ensure tallying succeeds
+		head := headers[len(headers)-1]
+
+		snap, err := New(&params.CliqueConfig{Epoch: tt.epoch}, db).snapshot(&testerChainReader{db: db}, head.Number.Uint64(), head.Hash(), headers)
+		if err != nil {
+			t.Errorf("test %d: failed to create voting snapshot: %v", i, err)
+			continue
+		}
+		// Verify the final list of signers against the expected ones
+		signers = make([]common.Address, len(tt.results))
+		for j, signer := range tt.results {
+			signers[j] = accounts.address(signer)
+		}
+		for j := 0; j < len(signers); j++ {
+			for k := j + 1; k < len(signers); k++ {
+				if bytes.Compare(signers[j][:], signers[k][:]) > 0 {
+					signers[j], signers[k] = signers[k], signers[j]
+				}
+			}
+		}
+		result := snap.signers()
+		if len(result) != len(signers) {
+			t.Errorf("test %d: signers mismatch: have %x, want %x", i, result, signers)
+			continue
+		}
+		for j := 0; j < len(result); j++ {
+			if !bytes.Equal(result[j][:], signers[j][:]) {
+				t.Errorf("test %d, signer %d: signer mismatch: have %x, want %x", i, j, result[j], signers[j])
+			}
+		}
+	}
+}

consensus/consensus.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package consensus implements different Ethereum consensus engines.
+package consensus
+
+import (
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+	"github.com/expanse-org/go-expanse/rpc"
+)
+
+// ChainReader defines a small collection of methods needed to access the local
+// blockchain during header and/or uncle verification.
+type ChainReader interface {
+	// Config retrieves the blockchain's chain configuration.
+	Config() *params.ChainConfig
+
+	// CurrentHeader retrieves the current header from the local chain.
+	CurrentHeader() *types.Header
+
+	// GetHeader retrieves a block header from the database by hash and number.
+	GetHeader(hash common.Hash, number uint64) *types.Header
+
+	// GetHeaderByNumber retrieves a block header from the database by number.
+	GetHeaderByNumber(number uint64) *types.Header
+
+	// GetHeaderByHash retrieves a block header from the database by its hash.
+	GetHeaderByHash(hash common.Hash) *types.Header
+
+	// GetBlock retrieves a block from the database by hash and number.
+	GetBlock(hash common.Hash, number uint64) *types.Block
+}
+
+// Engine is an algorithm agnostic consensus engine.
+type Engine interface {
+	// Author retrieves the Ethereum address of the account that minted the given
+	// block, which may be different from the header's coinbase if a consensus
+	// engine is based on signatures.
+	Author(header *types.Header) (common.Address, error)
+
+	// VerifyHeader checks whether a header conforms to the consensus rules of a
+	// given engine. Verifying the seal may be done optionally here, or explicitly
+	// via the VerifySeal method.
+	VerifyHeader(chain ChainReader, header *types.Header, seal bool) error
+
+	// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
+	// concurrently. The method returns a quit channel to abort the operations and
+	// a results channel to retrieve the async verifications (the order is that of
+	// the input slice).
+	VerifyHeaders(chain ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error)
+
+	// VerifyUncles verifies that the given block's uncles conform to the consensus
+	// rules of a given engine.
+	VerifyUncles(chain ChainReader, block *types.Block) error
+
+	// VerifySeal checks whether the crypto seal on a header is valid according to
+	// the consensus rules of the given engine.
+	VerifySeal(chain ChainReader, header *types.Header) error
+
+	// Prepare initializes the consensus fields of a block header according to the
+	// rules of a particular engine. The changes are executed inline.
+	Prepare(chain ChainReader, header *types.Header) error
+
+	// Finalize runs any post-transaction state modifications (e.g. block rewards)
+	// and assembles the final block.
+	//
+	// Note, the block header and state database might be updated to reflect any
+	// consensus rules that happen at finalization (e.g. block rewards).
+	Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction,
+		uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error)
+
+	// Seal generates a new block for the given input block with the local miner's
+	// seal place on top.
+	Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error)
+
+	// APIs returns the RPC APIs this consensus engine provides.
+	APIs(chain ChainReader) []rpc.API
+}
+
+// PoW is a consensus engine based on proof-of-work.
+type PoW interface {
+	Engine
+
+	// Hashrate returns the current mining hashrate of a PoW consensus engine.
+	Hashrate() float64
+}

consensus/errors.go

@@ -0,0 +1,33 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package consensus
+
+import "errors"
+
+var (
+	// ErrUnknownAncestor is returned when validating a block requires an ancestor
+	// that is unknown.
+	ErrUnknownAncestor = errors.New("unknown ancestor")
+
+	// ErrFutureBlock is returned when a block's timestamp is in the future according
+	// to the current node.
+	ErrFutureBlock = errors.New("block in the future")
+
+	// ErrInvalidNumber is returned if a block's number doesn't equal it's parent's
+	// plus one.
+	ErrInvalidNumber = errors.New("invalid block number")
+)

consensus/ethash/algorithm.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+package ethash
 
 import (
 	"encoding/binary"
@@ -27,6 +27,7 @@ import (
 	"unsafe"
 
 	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/common/bitutil"
 	"github.com/expanse-org/go-expanse/crypto"
 	"github.com/expanse-org/go-expanse/crypto/sha3"
 	"github.com/expanse-org/go-expanse/log"
@@ -142,7 +143,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) {
 				dstOff = j * hashBytes
 				xorOff = (binary.LittleEndian.Uint32(cache[dstOff:]) % uint32(rows)) * hashBytes
 			)
-			xorBytes(temp, cache[srcOff:srcOff+hashBytes], cache[xorOff:xorOff+hashBytes])
+			bitutil.XORBytes(temp, cache[srcOff:srcOff+hashBytes], cache[xorOff:xorOff+hashBytes])
 			keccak512(cache[dstOff:], temp)
 
 			atomic.AddUint32(&progress, 1)

consensus/ethash/algorithm_go1.7.go

@@ -16,7 +16,7 @@
 
 // +build !go1.8
 
-package pow
+package ethash
 
 // cacheSize calculates and returns the size of the ethash verification cache that
 // belongs to a certain block number. The cache size grows linearly, however, we

consensus/ethash/algorithm_go1.8.go

@@ -16,7 +16,7 @@
 
 // +build go1.8
 
-package pow
+package ethash
 
 import "math/big"
 

consensus/ethash/algorithm_go1.8_test.go

@@ -16,7 +16,7 @@
 
 // +build go1.8
 
-package pow
+package ethash
 
 import "testing"
 

consensus/ethash/algorithm_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+package ethash
 
 import (
 	"bytes"
@@ -704,8 +704,8 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 		go func(idx int) {
 			defer pend.Done()
 
-			ethash := NewFullEthash(cachedir, 0, 1, "", 0, 0)
+			ethash := New(cachedir, 0, 1, "", 0, 0)
-			if err := ethash.Verify(block); err != nil {
+			if err := ethash.VerifySeal(nil, block.Header()); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)
 			}
 		}(i)
@@ -713,17 +713,6 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 	pend.Wait()
 }
 
-func TestTestMode(t *testing.T) {
-	head := &types.Header{Difficulty: big.NewInt(100)}
-	ethash := NewTestEthash()
-	nonce, mix := ethash.Search(types.NewBlockWithHeader(head), nil)
-	head.Nonce = types.EncodeNonce(nonce)
-	copy(head.MixDigest[:], mix)
-	if err := ethash.Verify(types.NewBlockWithHeader(head)); err != nil {
-		t.Error("unexpected Verify error:", err)
-	}
-}
-
 // Benchmarks the cache generation performance.
 func BenchmarkCacheGeneration(b *testing.B) {
 	for i := 0; i < b.N; i++ {

consensus/ethash/consensus.go

@@ -0,0 +1,479 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/big"
+	"runtime"
+	"time"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/consensus/misc"
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+	set "gopkg.in/fatih/set.v0"
+)
+
+// Ethash proof-of-work protocol constants.
+var (
+	blockReward *big.Int = big.NewInt(8e+18) // Block reward in wei for successfully mining a block
+	maxUncles            = 2                 // Maximum number of uncles allowed in a single block
+)
+
+// Various error messages to mark blocks invalid. These should be private to
+// prevent engine specific errors from being referenced in the remainder of the
+// codebase, inherently breaking if the engine is swapped out. Please put common
+// error types into the consensus package.
+var (
+	errLargeBlockTime    = errors.New("timestamp too big")
+	errZeroBlockTime     = errors.New("timestamp equals parent's")
+	errTooManyUncles     = errors.New("too many uncles")
+	errDuplicateUncle    = errors.New("duplicate uncle")
+	errUncleIsAncestor   = errors.New("uncle is ancestor")
+	errDanglingUncle     = errors.New("uncle's parent is not ancestor")
+	errNonceOutOfRange   = errors.New("nonce out of range")
+	errInvalidDifficulty = errors.New("non-positive difficulty")
+	errInvalidMixDigest  = errors.New("invalid mix digest")
+	errInvalidPoW        = errors.New("invalid proof-of-work")
+)
+
+// Author implements consensus.Engine, returning the header's coinbase as the
+// proof-of-work verified author of the block.
+func (ethash *Ethash) Author(header *types.Header) (common.Address, error) {
+	return header.Coinbase, nil
+}
+
+// VerifyHeader checks whether a header conforms to the consensus rules of the
+// stock Ethereum ethash engine.
+func (ethash *Ethash) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull {
+		return nil
+	}
+	// Short circuit if the header is known, or it's parent not
+	number := header.Number.Uint64()
+	if chain.GetHeader(header.Hash(), number) != nil {
+		return nil
+	}
+	parent := chain.GetHeader(header.ParentHash, number-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	// Sanity checks passed, do a proper verification
+	return ethash.verifyHeader(chain, header, parent, false, seal)
+}
+
+// VerifyHeaders is similar to VerifyHeader, but verifies a batch of headers
+// concurrently. The method returns a quit channel to abort the operations and
+// a results channel to retrieve the async verifications.
+func (ethash *Ethash) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull || len(headers) == 0 {
+		abort, results := make(chan struct{}), make(chan error, len(headers))
+		for i := 0; i < len(headers); i++ {
+			results <- nil
+		}
+		return abort, results
+	}
+
+	// Spawn as many workers as allowed threads
+	workers := runtime.GOMAXPROCS(0)
+	if len(headers) < workers {
+		workers = len(headers)
+	}
+
+	// Create a task channel and spawn the verifiers
+	var (
+		inputs = make(chan int)
+		done   = make(chan int, workers)
+		errors = make([]error, len(headers))
+		abort  = make(chan struct{})
+	)
+	for i := 0; i < workers; i++ {
+		go func() {
+			for index := range inputs {
+				errors[index] = ethash.verifyHeaderWorker(chain, headers, seals, index)
+				done <- index
+			}
+		}()
+	}
+
+	errorsOut := make(chan error, len(headers))
+	go func() {
+		defer close(inputs)
+		var (
+			in, out = 0, 0
+			checked = make([]bool, len(headers))
+			inputs  = inputs
+		)
+		for {
+			select {
+			case inputs <- in:
+				if in++; in == len(headers) {
+					// Reached end of headers. Stop sending to workers.
+					inputs = nil
+				}
+			case index := <-done:
+				for checked[index] = true; checked[out]; out++ {
+					errorsOut <- errors[out]
+					if out == len(headers)-1 {
+						return
+					}
+				}
+			case <-abort:
+				return
+			}
+		}
+	}()
+	return abort, errorsOut
+}
+
+func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainReader, headers []*types.Header, seals []bool, index int) error {
+	var parent *types.Header
+	if index == 0 {
+		parent = chain.GetHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1)
+	} else if headers[index-1].Hash() == headers[index].ParentHash {
+		parent = headers[index-1]
+	}
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	if chain.GetHeader(headers[index].Hash(), headers[index].Number.Uint64()) != nil {
+		return nil // known block
+	}
+	return ethash.verifyHeader(chain, headers[index], parent, false, seals[index])
+}
+
+// VerifyUncles verifies that the given block's uncles conform to the consensus
+// rules of the stock Ethereum ethash engine.
+func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
+	// If we're running a full engine faking, accept any input as valid
+	if ethash.fakeFull {
+		return nil
+	}
+	// Verify that there are at most 2 uncles included in this block
+	if len(block.Uncles()) > maxUncles {
+		return errTooManyUncles
+	}
+	// Gather the set of past uncles and ancestors
+	uncles, ancestors := set.New(), make(map[common.Hash]*types.Header)
+
+	number, parent := block.NumberU64()-1, block.ParentHash()
+	for i := 0; i < 7; i++ {
+		ancestor := chain.GetBlock(parent, number)
+		if ancestor == nil {
+			break
+		}
+		ancestors[ancestor.Hash()] = ancestor.Header()
+		for _, uncle := range ancestor.Uncles() {
+			uncles.Add(uncle.Hash())
+		}
+		parent, number = ancestor.ParentHash(), number-1
+	}
+	ancestors[block.Hash()] = block.Header()
+	uncles.Add(block.Hash())
+
+	// Verify each of the uncles that it's recent, but not an ancestor
+	for _, uncle := range block.Uncles() {
+		// Make sure every uncle is rewarded only once
+		hash := uncle.Hash()
+		if uncles.Has(hash) {
+			return errDuplicateUncle
+		}
+		uncles.Add(hash)
+
+		// Make sure the uncle has a valid ancestry
+		if ancestors[hash] != nil {
+			return errUncleIsAncestor
+		}
+		if ancestors[uncle.ParentHash] == nil || uncle.ParentHash == block.ParentHash() {
+			return errDanglingUncle
+		}
+		if err := ethash.verifyHeader(chain, uncle, ancestors[uncle.ParentHash], true, true); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// verifyHeader checks whether a header conforms to the consensus rules of the
+// stock Ethereum ethash engine.
+//
+// See YP section 4.3.4. "Block Header Validity"
+func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent *types.Header, uncle bool, seal bool) error {
+	// Ensure that the header's extra-data section is of a reasonable size
+	if uint64(len(header.Extra)) > params.MaximumExtraDataSize {
+		return fmt.Errorf("extra-data too long: %d > %d", len(header.Extra), params.MaximumExtraDataSize)
+	}
+	// Verify the header's timestamp
+	if uncle {
+		if header.Time.Cmp(math.MaxBig256) > 0 {
+			return errLargeBlockTime
+		}
+	} else {
+		if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+			return consensus.ErrFutureBlock
+		}
+	}
+	if header.Time.Cmp(parent.Time) <= 0 {
+		return errZeroBlockTime
+	}
+	// Verify the block's difficulty based in it's timestamp and parent's difficulty
+	expected := CalcDifficulty(chain.Config(), header.Time.Uint64(), parent)
+	if expected.Cmp(header.Difficulty) != 0 {
+		return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, expected)
+	}
+	// Verify that the gas limit is <= 2^63-1
+	if header.GasLimit.Cmp(math.MaxBig63) > 0 {
+		return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, math.MaxBig63)
+	}
+	// Verify that the gasUsed is <= gasLimit
+	if header.GasUsed.Cmp(header.GasLimit) > 0 {
+		return fmt.Errorf("invalid gasUsed: have %v, gasLimit %v", header.GasUsed, header.GasLimit)
+	}
+
+	// Verify that the gas limit remains within allowed bounds
+	diff := new(big.Int).Set(parent.GasLimit)
+	diff = diff.Sub(diff, header.GasLimit)
+	diff.Abs(diff)
+
+	limit := new(big.Int).Set(parent.GasLimit)
+	limit = limit.Div(limit, params.GasLimitBoundDivisor)
+
+	if diff.Cmp(limit) >= 0 || header.GasLimit.Cmp(params.MinGasLimit) < 0 {
+		return fmt.Errorf("invalid gas limit: have %v, want %v += %v", header.GasLimit, parent.GasLimit, limit)
+	}
+	// Verify that the block number is parent's +1
+	if diff := new(big.Int).Sub(header.Number, parent.Number); diff.Cmp(big.NewInt(1)) != 0 {
+		return consensus.ErrInvalidNumber
+	}
+	// Verify the engine specific seal securing the block
+	if seal {
+		if err := ethash.VerifySeal(chain, header); err != nil {
+			return err
+		}
+	}
+	// If all checks passed, validate any special fields for hard forks
+	if err := misc.VerifyDAOHeaderExtraData(chain.Config(), header); err != nil {
+		return err
+	}
+	if err := misc.VerifyForkHashes(chain.Config(), header, uncle); err != nil {
+		return err
+	}
+	return nil
+}
+
+// CalcDifficulty is the difficulty adjustment algorithm. It returns
+// the difficulty that a new block should have when created at time
+// given the parent block's time and difficulty.
+//
+// TODO (karalabe): Move the chain maker into this package and make this private!
+func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int {
+	next := new(big.Int).Add(parent.Number, common.Big1)
+	switch {
+	case config.IsHomestead(next):
+		return calcDifficultyHomestead(time, parent)
+	default:
+		return calcDifficultyFrontier(time, parent)
+	}
+}
+
+// Some weird constants to avoid constant memory allocs for them.
+var (
+	expDiffPeriod = big.NewInt(100000)
+	big10         = big.NewInt(10)
+	big60         = big.NewInt(60)
+	bigMinus99    = big.NewInt(-99)
+)
+
+// calcDifficultyHomestead is the difficulty adjustment algorithm. It returns
+// the difficulty that a new block should have when created at time given the
+// parent block's time and difficulty. The calculation uses the Homestead rules.
+func calcDifficultyHomestead(time uint64, parent *types.Header) *big.Int {
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.mediawiki
+	// algorithm:
+	// diff = (parent_diff +
+	//         (parent_diff / 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
+	//        ) + 2^(periodCount - 2)
+
+	bigTime := new(big.Int).SetUint64(time)
+	bigParentTime := new(big.Int).Set(parent.Time)
+
+	// holds intermediate values to make the algo easier to read & audit
+	x := new(big.Int)
+	y := new(big.Int)
+
+	// 1 - (block_timestamp -parent_timestamp) // 10
+	x.Sub(bigTime, bigParentTime)
+	x.Div(x, big60)
+	x.Sub(common.Big1, x)
+
+	// max(1 - (block_timestamp - parent_timestamp) // 10, -99)))
+	if x.Cmp(bigMinus99) < 0 {
+		x.Set(bigMinus99)
+	}
+
+	// (parent_diff + parent_diff // 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99))
+	y.Div(parent.Difficulty, params.DifficultyBoundDivisor2)
+	x.Mul(y, x)
+	x.Add(parent.Difficulty, x)
+
+	// minimum difficulty can ever be (before exponential factor)
+	if x.Cmp(params.MinimumDifficulty) < 0 {
+		x.Set(params.MinimumDifficulty)
+	}
+
+	return x
+}
+
+// calcDifficultyFrontier is the difficulty adjustment algorithm. It returns the
+// difficulty that a new block should have when created at time given the parent
+// block's time and difficulty. The calculation uses the Frontier rules.
+func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int {
+	diff := new(big.Int)
+	adjust := new(big.Int)
+	bigTime := new(big.Int)
+	bigParentTime := new(big.Int)
+
+	if parent.Number.Cmp(params.HardFork1) < 0 {
+		adjust = new(big.Int).Div(parent.Difficulty, params.DifficultyBoundDivisor)
+	} else {
+		adjust = new(big.Int).Div(parent.Difficulty, params.DifficultyBoundDivisor2)
+	}
+
+	bigTime.SetUint64(time)
+	bigParentTime.Set(parent.Time)
+
+	if bigTime.Sub(bigTime, bigParentTime).Cmp(params.DurationLimit) < 0 {
+		diff.Add(parent.Difficulty, adjust)
+	} else {
+		diff.Sub(parent.Difficulty, adjust)
+	}
+	if diff.Cmp(params.MinimumDifficulty) < 0 {
+		diff.Set(params.MinimumDifficulty)
+	}
+
+	periodCount := new(big.Int).Add(parent.Number, common.Big1)
+	periodCount.Div(periodCount, expDiffPeriod)
+	if periodCount.Cmp(common.Big1) > 0 {
+		// diff = diff + 2^(periodCount - 2)
+		expDiff := periodCount.Sub(periodCount, common.Big2)
+		expDiff.Exp(common.Big2, expDiff, nil)
+		diff.Add(diff, expDiff)
+		diff = math.BigMax(diff, params.MinimumDifficulty)
+	}
+	return diff
+}
+
+// VerifySeal implements consensus.Engine, checking whether the given block satisfies
+// the PoW difficulty requirements.
+func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Header) error {
+	// If we're running a fake PoW, accept any seal as valid
+	if ethash.fakeMode {
+		time.Sleep(ethash.fakeDelay)
+		if ethash.fakeFail == header.Number.Uint64() {
+			return errInvalidPoW
+		}
+		return nil
+	}
+	// If we're running a shared PoW, delegate verification to it
+	if ethash.shared != nil {
+		return ethash.shared.VerifySeal(chain, header)
+	}
+	// Sanity check that the block number is below the lookup table size (60M blocks)
+	number := header.Number.Uint64()
+	if number/epochLength >= uint64(len(cacheSizes)) {
+		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
+		return errNonceOutOfRange
+	}
+	// Ensure that we have a valid difficulty for the block
+	if header.Difficulty.Sign() <= 0 {
+		return errInvalidDifficulty
+	}
+	// Recompute the digest and PoW value and verify against the header
+	cache := ethash.cache(number)
+
+	size := datasetSize(number)
+	if ethash.tester {
+		size = 32 * 1024
+	}
+	digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+	if !bytes.Equal(header.MixDigest[:], digest) {
+		return errInvalidMixDigest
+	}
+	target := new(big.Int).Div(maxUint256, header.Difficulty)
+	if new(big.Int).SetBytes(result).Cmp(target) > 0 {
+		return errInvalidPoW
+	}
+	return nil
+}
+
+// Prepare implements consensus.Engine, initializing the difficulty field of a
+// header to conform to the ethash protocol. The changes are done inline.
+func (ethash *Ethash) Prepare(chain consensus.ChainReader, header *types.Header) error {
+	parent := chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
+	if parent == nil {
+		return consensus.ErrUnknownAncestor
+	}
+	header.Difficulty = CalcDifficulty(chain.Config(), header.Time.Uint64(), parent)
+
+	return nil
+}
+
+// Finalize implements consensus.Engine, accumulating the block and uncle rewards,
+// setting the final state and assembling the block.
+func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
+	// Accumulate any block and uncle rewards and commit the final state root
+	AccumulateRewards(state, header, uncles)
+	header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
+
+	// Header seems complete, assemble into a block and return
+	return types.NewBlock(header, txs, uncles, receipts), nil
+}
+
+// Some weird constants to avoid constant memory allocs for them.
+var (
+	big8  = big.NewInt(8)
+	big32 = big.NewInt(32)
+)
+
+// AccumulateRewards credits the coinbase of the given block with the mining
+// reward. The total reward consists of the static block reward and rewards for
+// included uncles. The coinbase of each uncle block is also rewarded.
+//
+// TODO (karalabe): Move the chain maker into this package and make this private!
+func AccumulateRewards(state *state.StateDB, header *types.Header, uncles []*types.Header) {
+	reward := new(big.Int).Set(blockReward)
+	r := new(big.Int)
+	for _, uncle := range uncles {
+		r.Add(uncle.Number, big8)
+		r.Sub(r, header.Number)
+		r.Mul(r, blockReward)
+		r.Div(r, big8)
+		state.AddBalance(uncle.Coinbase, r)
+
+		r.Div(blockReward, big32)
+		reward.Add(reward, r)
+	}
+	state.AddBalance(header.Coinbase, reward)
+}

consensus/ethash/consensus_test.go

@@ -0,0 +1,84 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"encoding/json"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/common/math"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+type diffTest struct {
+	ParentTimestamp    uint64
+	ParentDifficulty   *big.Int
+	CurrentTimestamp   uint64
+	CurrentBlocknumber *big.Int
+	CurrentDifficulty  *big.Int
+}
+
+func (d *diffTest) UnmarshalJSON(b []byte) (err error) {
+	var ext struct {
+		ParentTimestamp    string
+		ParentDifficulty   string
+		CurrentTimestamp   string
+		CurrentBlocknumber string
+		CurrentDifficulty  string
+	}
+	if err := json.Unmarshal(b, &ext); err != nil {
+		return err
+	}
+
+	d.ParentTimestamp = math.MustParseUint64(ext.ParentTimestamp)
+	d.ParentDifficulty = math.MustParseBig256(ext.ParentDifficulty)
+	d.CurrentTimestamp = math.MustParseUint64(ext.CurrentTimestamp)
+	d.CurrentBlocknumber = math.MustParseBig256(ext.CurrentBlocknumber)
+	d.CurrentDifficulty = math.MustParseBig256(ext.CurrentDifficulty)
+
+	return nil
+}
+
+func TestCalcDifficulty(t *testing.T) {
+	file, err := os.Open("../../tests/files/BasicTests/difficulty.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	tests := make(map[string]diffTest)
+	err = json.NewDecoder(file).Decode(&tests)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := &params.ChainConfig{HomesteadBlock: big.NewInt(1150000)}
+	for name, test := range tests {
+		number := new(big.Int).Sub(test.CurrentBlocknumber, big.NewInt(1))
+		diff := CalcDifficulty(config, test.CurrentTimestamp, &types.Header{
+			Number:     number,
+			Time:       new(big.Int).SetUint64(test.ParentTimestamp),
+			Difficulty: test.ParentDifficulty,
+		})
+		if diff.Cmp(test.CurrentDifficulty) != 0 {
+			t.Error(name, "failed. Expected", test.CurrentDifficulty, "and calculated", diff)
+		}
+	}
+}

consensus/ethash/ethash.go

@@ -14,10 +14,10 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package pow
+// Package ethash implements the ethash proof-of-work consensus engine.
+package ethash
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"math"
@@ -32,24 +32,21 @@ import (
 	"unsafe"
 
 	mmap "github.com/edsrzf/mmap-go"
+
+	"github.com/expanse-org/go-expanse/consensus"
 	"github.com/expanse-org/go-expanse/log"
+	"github.com/expanse-org/go-expanse/rpc"
 	metrics "github.com/rcrowley/go-metrics"
 )
 
-var (
+var ErrInvalidDumpMagic = errors.New("invalid dump magic")
-	ErrInvalidDumpMagic  = errors.New("invalid dump magic")
-	ErrNonceOutOfRange   = errors.New("nonce out of range")
-	ErrInvalidDifficulty = errors.New("non-positive difficulty")
-	ErrInvalidMixDigest  = errors.New("invalid mix digest")
-	ErrInvalidPoW        = errors.New("pow difficulty invalid")
-)
 
 var (
 	// maxUint256 is a big integer representing 2^256-1
 	maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = NewFullEthash("", 3, 0, "", 1, 0)
+	sharedEthash = New("", 3, 0, "", 1, 0)
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -134,13 +131,16 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint
 	data := buffer[len(dumpMagic):]
 	generator(data)
 
-	if err := mem.Flush(); err != nil {
+	if err := mem.Unmap(); err != nil {
-		mem.Unmap()
-		dump.Close()
 		return nil, nil, nil, err
 	}
-	os.Rename(temp, path)
+	if err := dump.Close(); err != nil {
-	return dump, mem, data, nil
+		return nil, nil, nil, err
+	}
+	if err := os.Rename(temp, path); err != nil {
+		return nil, nil, nil, err
+	}
+	return memoryMap(path)
 }
 
 // cache wraps an ethash cache with some metadata to allow easier concurrent use.
@@ -321,7 +321,8 @@ func MakeDataset(block uint64, dir string) {
 	d.release()
 }
 
-// Ethash is a PoW data struture implementing the ethash algorithm.
+// Ethash is a consensus engine based on proot-of-work implementing the ethash
+// algorithm.
 type Ethash struct {
 	cachedir     string // Data directory to store the verification caches
 	cachesinmem  int    // Number of caches to keep in memory
@@ -334,15 +335,26 @@ type Ethash struct {
 	fcache   *cache              // Pre-generated cache for the estimated future epoch
 	datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often
 	fdataset *dataset            // Pre-generated dataset for the estimated future epoch
-	lock     sync.Mutex          // Ensures thread safety for the in-memory caches
 
+	// Mining related fields
+	rand     *rand.Rand    // Properly seeded random source for nonces
+	threads  int           // Number of threads to mine on if mining
+	update   chan struct{} // Notification channel to update mining parameters
 	hashrate metrics.Meter // Meter tracking the average hashrate
 
-	tester bool // Flag whether to use a smaller test dataset
+	// The fields below are hooks for testing
+	tester    bool          // Flag whether to use a smaller test dataset
+	shared    *Ethash       // Shared PoW verifier to avoid cache regeneration
+	fakeMode  bool          // Flag whether to disable PoW checking
+	fakeFull  bool          // Flag whether to disable all consensus rules
+	fakeFail  uint64        // Block number which fails PoW check even in fake mode
+	fakeDelay time.Duration // Time delay to sleep for before returning from verify
+
+	lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields
 }
 
-// NewFullEthash creates a full sized ethash PoW scheme.
+// New creates a full sized ethash PoW scheme.
-func NewFullEthash(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) PoW {
+func New(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) *Ethash {
 	if cachesinmem <= 0 {
 		log.Warn("One ethash cache must alwast be in memory", "requested", cachesinmem)
 		cachesinmem = 1
@@ -362,58 +374,55 @@ func NewFullEthash(cachedir string, cachesinmem, cachesondisk int, dagdir string
 		dagsondisk:   dagsondisk,
 		caches:       make(map[uint64]*cache),
 		datasets:     make(map[uint64]*dataset),
+		update:       make(chan struct{}),
 		hashrate:     metrics.NewMeter(),
 	}
 }
 
-// NewTestEthash creates a small sized ethash PoW scheme useful only for testing
+// NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
-func NewTestEthash() PoW {
+func NewTester() *Ethash {
 	return &Ethash{
 		cachesinmem: 1,
 		caches:      make(map[uint64]*cache),
 		datasets:    make(map[uint64]*dataset),
 		tester:      true,
+		update:      make(chan struct{}),
 		hashrate:    metrics.NewMeter(),
 	}
 }
 
-// NewSharedEthash creates a full sized ethash PoW shared between all requesters
+// NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts
-// running in the same process.
+// all blocks' seal as valid, though they still have to conform to the Ethereum
-func NewSharedEthash() PoW {
+// consensus rules.
-	return sharedEthash
+func NewFaker() *Ethash {
+	return &Ethash{fakeMode: true}
 }
 
-// Verify implements PoW, checking whether the given block satisfies the PoW
+// NewFakeFailer creates a ethash consensus engine with a fake PoW scheme that
-// difficulty requirements.
+// accepts all blocks as valid apart from the single one specified, though they
-func (ethash *Ethash) Verify(block Block) error {
+// still have to conform to the Ethereum consensus rules.
-	// Sanity check that the block number is below the lookup table size (60M blocks)
+func NewFakeFailer(fail uint64) *Ethash {
-	number := block.NumberU64()
+	return &Ethash{fakeMode: true, fakeFail: fail}
-	if number/epochLength >= uint64(len(cacheSizes)) {
+}
-		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
-		return ErrNonceOutOfRange
-	}
-	// Ensure that we have a valid difficulty for the block
-	difficulty := block.Difficulty()
-	if difficulty.Sign() <= 0 {
-		return ErrInvalidDifficulty
-	}
-	// Recompute the digest and PoW value and verify against the block
-	cache := ethash.cache(number)
 
-	size := datasetSize(number)
+// NewFakeDelayer creates a ethash consensus engine with a fake PoW scheme that
-	if ethash.tester {
+// accepts all blocks as valid, but delays verifications by some time, though
-		size = 32 * 1024
+// they still have to conform to the Ethereum consensus rules.
-	}
+func NewFakeDelayer(delay time.Duration) *Ethash {
-	digest, result := hashimotoLight(size, cache, block.HashNoNonce().Bytes(), block.Nonce())
+	return &Ethash{fakeMode: true, fakeDelay: delay}
-	if !bytes.Equal(block.MixDigest().Bytes(), digest) {
+}
-		return ErrInvalidMixDigest
+
-	}
+// NewFullFaker creates a ethash consensus engine with a full fake scheme that
-	target := new(big.Int).Div(maxUint256, difficulty)
+// accepts all blocks as valid, without checking any consensus rules whatsoever.
-	if new(big.Int).SetBytes(result).Cmp(target) > 0 {
+func NewFullFaker() *Ethash {
-		return ErrInvalidPoW
+	return &Ethash{fakeMode: true, fakeFull: true}
-	}
+}
-	return nil
+
+// NewShared creates a full sized ethash PoW shared between all requesters running
+// in the same process.
+func NewShared() *Ethash {
+	return &Ethash{shared: sharedEthash}
 }
 
 // cache tries to retrieve a verification cache for the specified block number
@@ -459,8 +468,9 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 			future = &cache{epoch: epoch + 1}
 			ethash.fcache = future
 		}
+		// New current cache, set its initial timestamp
+		current.used = time.Now()
 	}
-	current.used = time.Now()
 	ethash.lock.Unlock()
 
 	// Wait for generation finish, bump the timestamp and finalize the cache
@@ -477,43 +487,6 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 	return current.cache
 }
 
-// Search implements PoW, attempting to find a nonce that satisfies the block's
-// difficulty requirements.
-func (ethash *Ethash) Search(block Block, stop <-chan struct{}) (uint64, []byte) {
-	var (
-		hash     = block.HashNoNonce().Bytes()
-		diff     = block.Difficulty()
-		target   = new(big.Int).Div(maxUint256, diff)
-		dataset  = ethash.dataset(block.NumberU64())
-		rand     = rand.New(rand.NewSource(time.Now().UnixNano()))
-		nonce    = uint64(rand.Int63())
-		attempts int64
-	)
-	// Start generating random nonces until we abort or find a good one
-	for {
-		select {
-		case <-stop:
-			// Mining terminated, update stats and abort
-			ethash.hashrate.Mark(attempts)
-			return 0, nil
-
-		default:
-			// We don't have to update hash rate on every nonce, so update after after 2^X nonces
-			attempts++
-			if (attempts % (1 << 15)) == 0 {
-				ethash.hashrate.Mark(attempts)
-				attempts = 0
-			}
-			// Compute the PoW value of this nonce
-			digest, result := hashimotoFull(dataset, hash, nonce)
-			if new(big.Int).SetBytes(result).Cmp(target) <= 0 {
-				return nonce, digest
-			}
-			nonce++
-		}
-	}
-}
-
 // dataset tries to retrieve a mining dataset for the specified block number
 // by first checking against a list of in-memory datasets, then against DAGs
 // stored on disk, and finally generating one if none can be found.
@@ -558,8 +531,9 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 			future = &dataset{epoch: epoch + 1}
 			ethash.fdataset = future
 		}
+		// New current dataset, set its initial timestamp
+		current.used = time.Now()
 	}
-	current.used = time.Now()
 	ethash.lock.Unlock()
 
 	// Wait for generation finish, bump the timestamp and finalize the cache
@@ -576,14 +550,51 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 	return current.dataset
 }
 
+// Threads returns the number of mining threads currently enabled. This doesn't
+// necessarily mean that mining is running!
+func (ethash *Ethash) Threads() int {
+	ethash.lock.Lock()
+	defer ethash.lock.Unlock()
+
+	return ethash.threads
+}
+
+// SetThreads updates the number of mining threads currently enabled. Calling
+// this method does not start mining, only sets the thread count. If zero is
+// specified, the miner will use all cores of the machine. Setting a thread
+// count below zero is allowed and will cause the miner to idle, without any
+// work being done.
+func (ethash *Ethash) SetThreads(threads int) {
+	ethash.lock.Lock()
+	defer ethash.lock.Unlock()
+
+	// If we're running a shared PoW, set the thread count on that instead
+	if ethash.shared != nil {
+		ethash.shared.SetThreads(threads)
+		return
+	}
+	// Update the threads and ping any running seal to pull in any changes
+	ethash.threads = threads
+	select {
+	case ethash.update <- struct{}{}:
+	default:
+	}
+}
+
 // Hashrate implements PoW, returning the measured rate of the search invocations
 // per second over the last minute.
 func (ethash *Ethash) Hashrate() float64 {
 	return ethash.hashrate.Rate1()
 }
 
-// EthashSeedHash is the seed to use for generating a vrification cache and the
+// APIs implements consensus.Engine, returning the user facing RPC APIs. Currently
-// mining dataset.
+// that is empty.
-func EthashSeedHash(block uint64) []byte {
+func (ethash *Ethash) APIs(chain consensus.ChainReader) []rpc.API {
+	return nil
+}
+
+// SeedHash is the seed to use for generating a verification cache and the mining
+// dataset.
+func SeedHash(block uint64) []byte {
 	return seedHash(block)
 }

consensus/ethash/ethash_test.go

@@ -0,0 +1,40 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/expanse-org/go-expanse/core/types"
+)
+
+// Tests that ethash works correctly in test mode.
+func TestTestMode(t *testing.T) {
+	head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+
+	ethash := NewTester()
+	block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil)
+	if err != nil {
+		t.Fatalf("failed to seal block: %v", err)
+	}
+	head.Nonce = types.EncodeNonce(block.Nonce())
+	head.MixDigest = block.MixDigest()
+	if err := ethash.VerifySeal(nil, head); err != nil {
+		t.Fatalf("unexpected verification error: %v", err)
+	}
+}

consensus/ethash/sealer.go

@@ -0,0 +1,149 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethash
+
+import (
+	crand "crypto/rand"
+	"math"
+	"math/big"
+	"math/rand"
+	"runtime"
+	"sync"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/consensus"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/log"
+)
+
+// Seal implements consensus.Engine, attempting to find a nonce that satisfies
+// the block's difficulty requirements.
+func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+	// If we're running a fake PoW, simply return a 0 nonce immediately
+	if ethash.fakeMode {
+		header := block.Header()
+		header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{}
+		return block.WithSeal(header), nil
+	}
+	// If we're running a shared PoW, delegate sealing to it
+	if ethash.shared != nil {
+		return ethash.shared.Seal(chain, block, stop)
+	}
+	// Create a runner and the multiple search threads it directs
+	abort := make(chan struct{})
+	found := make(chan *types.Block)
+
+	ethash.lock.Lock()
+	threads := ethash.threads
+	if ethash.rand == nil {
+		seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
+		if err != nil {
+			ethash.lock.Unlock()
+			return nil, err
+		}
+		ethash.rand = rand.New(rand.NewSource(seed.Int64()))
+	}
+	ethash.lock.Unlock()
+	if threads == 0 {
+		threads = runtime.NumCPU()
+	}
+	if threads < 0 {
+		threads = 0 // Allows disabling local mining without extra logic around local/remote
+	}
+	var pend sync.WaitGroup
+	for i := 0; i < threads; i++ {
+		pend.Add(1)
+		go func(id int, nonce uint64) {
+			defer pend.Done()
+			ethash.mine(block, id, nonce, abort, found)
+		}(i, uint64(ethash.rand.Int63()))
+	}
+	// Wait until sealing is terminated or a nonce is found
+	var result *types.Block
+	select {
+	case <-stop:
+		// Outside abort, stop all miner threads
+		close(abort)
+	case result = <-found:
+		// One of the threads found a block, abort all others
+		close(abort)
+	case <-ethash.update:
+		// Thread count was changed on user request, restart
+		close(abort)
+		pend.Wait()
+		return ethash.Seal(chain, block, stop)
+	}
+	// Wait for all miners to terminate and return the block
+	pend.Wait()
+	return result, nil
+}
+
+// mine is the actual proof-of-work miner that searches for a nonce starting from
+// seed that results in correct final block difficulty.
+func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) {
+	// Extract some data from the header
+	var (
+		header = block.Header()
+		hash   = header.HashNoNonce().Bytes()
+		target = new(big.Int).Div(maxUint256, header.Difficulty)
+
+		number  = header.Number.Uint64()
+		dataset = ethash.dataset(number)
+	)
+	// Start generating random nonces until we abort or find a good one
+	var (
+		attempts = int64(0)
+		nonce    = seed
+	)
+	logger := log.New("miner", id)
+	logger.Trace("Started ethash search for new nonces", "seed", seed)
+	for {
+		select {
+		case <-abort:
+			// Mining terminated, update stats and abort
+			logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed)
+			ethash.hashrate.Mark(attempts)
+			return
+
+		default:
+			// We don't have to update hash rate on every nonce, so update after after 2^X nonces
+			attempts++
+			if (attempts % (1 << 15)) == 0 {
+				ethash.hashrate.Mark(attempts)
+				attempts = 0
+			}
+			// Compute the PoW value of this nonce
+			digest, result := hashimotoFull(dataset, hash, nonce)
+			if new(big.Int).SetBytes(result).Cmp(target) <= 0 {
+				// Correct nonce found, create a new header with it
+				header = types.CopyHeader(header)
+				header.Nonce = types.EncodeNonce(nonce)
+				header.MixDigest = common.BytesToHash(digest)
+
+				// Seal and return a block (if still needed)
+				select {
+				case found <- block.WithSeal(header):
+					logger.Trace("Ethash nonce found and reported", "attempts", nonce-seed, "nonce", nonce)
+				case <-abort:
+					logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce)
+				}
+				return
+			}
+			nonce++
+		}
+	}
+}

consensus/misc/dao.go

@@ -0,0 +1,85 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package misc
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+
+	"github.com/expanse-org/go-expanse/core/state"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+var (
+	// ErrBadProDAOExtra is returned if a header doens't support the DAO fork on a
+	// pro-fork client.
+	ErrBadProDAOExtra = errors.New("bad DAO pro-fork extra-data")
+
+	// ErrBadNoDAOExtra is returned if a header does support the DAO fork on a no-
+	// fork client.
+	ErrBadNoDAOExtra = errors.New("bad DAO no-fork extra-data")
+)
+
+// VerifyDAOHeaderExtraData validates the extra-data field of a block header to
+// ensure it conforms to DAO hard-fork rules.
+//
+// DAO hard-fork extension to the header validity:
+//   a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range
+//      with the fork specific extra-data set
+//   b) if the node is pro-fork, require blocks in the specific range to have the
+//      unique extra-data set.
+func VerifyDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error {
+	// Short circuit validation if the node doesn't care about the DAO fork
+	if config.DAOForkBlock == nil {
+		return nil
+	}
+	// Make sure the block is within the fork's modified extra-data range
+	limit := new(big.Int).Add(config.DAOForkBlock, params.DAOForkExtraRange)
+	if header.Number.Cmp(config.DAOForkBlock) < 0 || header.Number.Cmp(limit) >= 0 {
+		return nil
+	}
+	// Depending whether we support or oppose the fork, validate the extra-data contents
+	if config.DAOForkSupport {
+		if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
+			return ErrBadProDAOExtra
+		}
+	} else {
+		if bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
+			return ErrBadNoDAOExtra
+		}
+	}
+	// All ok, header has the same extra-data we expect
+	return nil
+}
+
+// ApplyDAOHardFork modifies the state database according to the DAO hard-fork
+// rules, transferring all balances of a set of DAO accounts to a single refund
+// contract.
+func ApplyDAOHardFork(statedb *state.StateDB) {
+	// Retrieve the contract to refund balances into
+	if !statedb.Exist(params.DAORefundContract) {
+		statedb.CreateAccount(params.DAORefundContract)
+	}
+
+	// Move every DAO account and extra-balance account funds into the refund contract
+	for _, addr := range params.DAODrainList() {
+		statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr))
+		statedb.SetBalance(addr, new(big.Int))
+	}
+}

consensus/misc/forks.go

@@ -0,0 +1,43 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package misc
+
+import (
+	"fmt"
+
+	"github.com/expanse-org/go-expanse/common"
+	"github.com/expanse-org/go-expanse/core/types"
+	"github.com/expanse-org/go-expanse/params"
+)
+
+// VerifyForkHashes verifies that blocks conforming to network hard-forks do have
+// the correct hashes, to avoid clients going off on different chains. This is an
+// optional feature.
+func VerifyForkHashes(config *params.ChainConfig, header *types.Header, uncle bool) error {
+	// We don't care about uncles
+	if uncle {
+		return nil
+	}
+	// If the homestead reprice hash is set, validate it
+	if config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 {
+		if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() {
+			return fmt.Errorf("homestead gas reprice fork: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash)
+		}
+	}
+	// All ok, return
+	return nil
+}

console/bridge.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"github.com/expanse-org/go-expanse/log"
@@ -240,17 +241,19 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) {
 		throwJSException(err.Error())
 	}
 	var (
-		rawReq = []byte(reqVal.String())
+		rawReq = reqVal.String()
+		dec    = json.NewDecoder(strings.NewReader(rawReq))
 		reqs   []jsonrpcCall
 		batch  bool
 	)
+	dec.UseNumber() // avoid float64s
 	if rawReq[0] == '[' {
 		batch = true
-		json.Unmarshal(rawReq, &reqs)
+		dec.Decode(&reqs)
 	} else {
 		batch = false
 		reqs = make([]jsonrpcCall, 1)
-		json.Unmarshal(rawReq, &reqs[0])
+		dec.Decode(&reqs[0])
 	}
 
 	// Execute the requests.

console/console_test.go

@@ -91,7 +91,7 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester {
 	}
 
 	// Create a networkless protocol stack and start an Ethereum service within
-	stack, err := node.New(&node.Config{DataDir: workspace, UseLightweightKDF: true, Name: testInstance, NoDiscovery: true})
+	stack, err := node.New(&node.Config{DataDir: workspace, UseLightweightKDF: true, Name: testInstance})
 	if err != nil {
 		t.Fatalf("failed to create node: %v", err)
 	}

containers/docker/master-alpine/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.5
 
 RUN \
   apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
-  git clone --depth 1 --branch release/1.5 https://github.com/expanse-org/go-expanse && \
+  git clone --depth 1 --branch release/1.6 https://github.com/expanse-org/go-expanse && \
   (cd go-expanse && make gexp) && \
   cp go-expanse/build/bin/gexp /gexp && \
   apk del go git make gcc musl-dev linux-headers && \

containers/vagrant/.gitignore

@@ -0,0 +1 @@
+.vagrant