Merge branch 'master' of https://github.com/ethereum/go-ethereum

.dockerignore

@@ -1,4 +1,7 @@
 **/.git
+.git
+!.git/HEAD
+!.git/refs/heads
 **/*_test.go
 
 build/_workspace

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+accounts/usbwallet @karalabe
+consensus          @karalabe
+core/              @karalabe @holiman
+eth/               @karalabe
+mobile/            @karalabe
+p2p/               @fjl @zsfelfoldi

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,9 @@
+Hi there,
+
+please note that this is an issue tracker reserved for bug reports and feature requests.
+
+For general questions please use the gitter channel or the Ethereum stack exchange at https://ethereum.stackexchange.com.
+
 #### System information
 
 Geth version: `geth version`

.gitignore

@@ -33,3 +33,8 @@ profile.cov
 
 # IdeaIDE
 .idea
+
+# dashboard
+/dashboard/assets/node_modules
+/dashboard/assets/stats.json
+/dashboard/assets/public/bundle.js

.travis.yml

@@ -38,7 +38,7 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage
 
     - os: osx
       go: 1.9.x
@@ -48,7 +48,21 @@ matrix:
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage
+
+    # This builder only tests code linters on latest version of Go
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.9.x
+      env:
+        - lint
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
@@ -133,16 +147,16 @@ matrix:
         - azure-android
         - maven-android
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -o android-ndk-r14b.zip
+        - curl https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip -o android-ndk-r15c.zip
-        - unzip -q android-ndk-r14b.zip && rm android-ndk-r14b.zip
+        - unzip -q android-ndk-r15c.zip && rm android-ndk-r15c.zip
-        - mv android-ndk-r14b $HOME
+        - mv android-ndk-r15c $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r14b
+        - export ANDROID_NDK=$HOME/android-ndk-r15c
 
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum

Dockerfile

@@ -12,5 +12,5 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
-EXPOSE 8545 8546 30303 30303/udp
+EXPOSE 8545 8546 30303 30303/udp 30304/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -0,0 +1,15 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.9-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make all
+
+# Pull all binaries into a second stage deploy alpine container
+FROM alpine:latest
+
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
+
+EXPOSE 8545 8546 30303 30303/udp 30304/udp

README.md

@@ -35,7 +35,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/EthereumCommonwealth/go-callisto/wiki/Command-Line-Options) for command line options. |
 | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/EthereumCommonwealth/go-callisto/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
-| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
+| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
 | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
 | `swarm`    | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
@@ -130,6 +130,8 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
 
 This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does.  It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
 
+Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
+
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
@@ -264,7 +266,7 @@ instance for mining, run it with all your usual flags, extended by:
 $ geth <usual-flags> --mine --minerthreads=1 --etherbase=0x0000000000000000000000000000000000000000
 ```
 
-Which will start mining bocks and transactions on a single CPU thread, crediting all proceedings to
+Which will start mining blocks and transactions on a single CPU thread, crediting all proceedings to
 the account specified by `--etherbase`. You can further tune the mining by changing the default gas
 limit blocks converge to (`--targetgaslimit`) and the price transactions are accepted at (`--gasprice`).
 

VERSION

@@ -1 +1 @@
-1.7.2
+1.8.0

accounts/abi/abi.go

@@ -20,10 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"reflect"
-	"strings"
-
-	"github.com/EthereumCommonwealth/go-callisto/common"
 )
 
 // The ABI holds information about a contract's context and available
@@ -76,106 +72,27 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
 	return append(method.Id(), arguments...), nil
 }
 
-// these variable are used to determine certain types during type assertion for
-// assignment.
-var (
-	r_interSlice = reflect.TypeOf([]interface{}{})
-	r_hash       = reflect.TypeOf(common.Hash{})
-	r_bytes      = reflect.TypeOf([]byte{})
-	r_byte       = reflect.TypeOf(byte(0))
-)
-
 // Unpack output in v according to the abi specification
-func (abi ABI) Unpack(v interface{}, name string, output []byte) error {
+func (abi ABI) Unpack(v interface{}, name string, output []byte) (err error) {
-	var method = abi.Methods[name]
+	if err = bytesAreProper(output); err != nil {
-
+		return err
-	if len(output) == 0 {
-		return fmt.Errorf("abi: unmarshalling empty output")
 	}
-
+	// since there can't be naming collisions with contracts and events,
-	// make sure the passed value is a pointer
+	// we need to decide whether we're calling a method or an event
-	valueOf := reflect.ValueOf(v)
+	var unpack unpacker
-	if reflect.Ptr != valueOf.Kind() {
+	if method, ok := abi.Methods[name]; ok {
-		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+		unpack = method
-	}
+	} else if event, ok := abi.Events[name]; ok {
-
+		unpack = event
-	var (
-		value = valueOf.Elem()
-		typ   = value.Type()
-	)
-
-	if len(method.Outputs) > 1 {
-		switch value.Kind() {
-		// struct will match named return values to the struct's field
-		// names
-		case reflect.Struct:
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				reflectValue := reflect.ValueOf(marshalledValue)
-
-				for j := 0; j < typ.NumField(); j++ {
-					field := typ.Field(j)
-					// TODO read tags: `abi:"fieldName"`
-					if field.Name == strings.ToUpper(method.Outputs[i].Name[:1])+method.Outputs[i].Name[1:] {
-						if err := set(value.Field(j), reflectValue, method.Outputs[i]); err != nil {
-							return err
-						}
-					}
-				}
-			}
-		case reflect.Slice:
-			if !value.Type().AssignableTo(r_interSlice) {
-				return fmt.Errorf("abi: cannot marshal tuple in to slice %T (only []interface{} is supported)", v)
-			}
-
-			// if the slice already contains values, set those instead of the interface slice itself.
-			if value.Len() > 0 {
-				if len(method.Outputs) > value.Len() {
-					return fmt.Errorf("abi: cannot marshal in to slices of unequal size (require: %v, got: %v)", len(method.Outputs), value.Len())
-				}
-
-				for i := 0; i < len(method.Outputs); i++ {
-					marshalledValue, err := toGoType(i, method.Outputs[i], output)
-					if err != nil {
-						return err
-					}
-					reflectValue := reflect.ValueOf(marshalledValue)
-					if err := set(value.Index(i).Elem(), reflectValue, method.Outputs[i]); err != nil {
-						return err
-					}
-				}
-				return nil
-			}
-
-			// create a new slice and start appending the unmarshalled
-			// values to the new interface slice.
-			z := reflect.MakeSlice(typ, 0, len(method.Outputs))
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				z = reflect.Append(z, reflect.ValueOf(marshalledValue))
-			}
-			value.Set(z)
-		default:
-			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
-		}
-
 	} else {
-		marshalledValue, err := toGoType(0, method.Outputs[0], output)
+		return fmt.Errorf("abi: could not locate named method or event.")
-		if err != nil {
-			return err
-		}
-		if err := set(value, reflect.ValueOf(marshalledValue), method.Outputs[0]); err != nil {
-			return err
-		}
 	}
 
-	return nil
+	// requires a struct to unpack into for a tuple return...
+	if unpack.isTupleReturn() {
+		return unpack.tupleUnpack(v, output)
+	}
+	return unpack.singleUnpack(v, output)
 }
 
 func (abi *ABI) UnmarshalJSON(data []byte) error {

accounts/abi/abi_test.go

@@ -29,25 +29,6 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/crypto"
 )
 
-// formatSilceOutput add padding to the value and adds a size
-func formatSliceOutput(v ...[]byte) []byte {
-	off := common.LeftPadBytes(big.NewInt(int64(len(v))).Bytes(), 32)
-	output := append(off, make([]byte, 0, len(v)*32)...)
-
-	for _, value := range v {
-		output = append(output, common.LeftPadBytes(value, 32)...)
-	}
-	return output
-}
-
-// quick helper padding
-func pad(input []byte, size int, left bool) []byte {
-	if left {
-		return common.LeftPadBytes(input, size)
-	}
-	return common.RightPadBytes(input, size)
-}
-
 const jsondata = `
 [
 	{ "type" : "function", "name" : "balance", "constant" : true },
@@ -191,7 +172,7 @@ func TestMethodSignature(t *testing.T) {
 		t.Errorf("expected ids to match %x != %x", m.Id(), idexp)
 	}
 
-	uintt, _ := NewType("uint")
+	uintt, _ := NewType("uint256")
 	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {

accounts/abi/bind/backends/simulated.go

@@ -41,6 +41,7 @@ import (
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
 var errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block")
+var errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction")
 
 // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in
 // the background. Its main purpose is to allow easily testing contract bindings.
@@ -59,7 +60,7 @@ type SimulatedBackend struct {
 // for testing purposes.
 func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
-	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
+	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
 	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
@@ -203,32 +204,46 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	// Binary search the gas requirement, as it may be higher than the amount used
+	// Determine the lowest and highest possible gas limits to binary search in between
 	var (
-		lo uint64 = params.TxGas - 1
+		lo  uint64 = params.TxGas - 1
-		hi uint64
+		hi  uint64
+		cap uint64
 	)
 	if call.Gas != nil && call.Gas.Uint64() >= params.TxGas {
 		hi = call.Gas.Uint64()
 	} else {
 		hi = b.pendingBlock.GasLimit().Uint64()
 	}
-	for lo+1 < hi {
+	cap = hi
-		// Take a guess at the gas, and check transaction validity
+
-		mid := (hi + lo) / 2
+	// Create a helper to check if a gas allowance results in an executable transaction
-		call.Gas = new(big.Int).SetUint64(mid)
+	executable := func(gas uint64) bool {
+		call.Gas = new(big.Int).SetUint64(gas)
 
 		snapshot := b.pendingState.Snapshot()
 		_, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
 		b.pendingState.RevertToSnapshot(snapshot)
 
-		// If the transaction became invalid or execution failed, raise the gas limit
 		if err != nil || failed {
-			lo = mid
+			return false
-			continue
+		}
+		return true
+	}
+	// Execute the binary search and hone in on an executable gas limit
+	for lo+1 < hi {
+		mid := (hi + lo) / 2
+		if !executable(mid) {
+			lo = mid
+		} else {
+			hi = mid
+		}
+	}
+	// Reject the transaction as invalid if it still fails at the highest allowance
+	if hi == cap {
+		if !executable(hi) {
+			return nil, errGasEstimationFailed
 		}
-		// Otherwise assume the transaction succeeded, lower the gas limit
-		hi = mid
 	}
 	return new(big.Int).SetUint64(hi), nil
 }

accounts/abi/error.go

@@ -39,22 +39,23 @@ func formatSliceString(kind reflect.Kind, sliceSize int) string {
 // type in t.
 func sliceTypeCheck(t Type, val reflect.Value) error {
 	if val.Kind() != reflect.Slice && val.Kind() != reflect.Array {
-		return typeErr(formatSliceString(t.Kind, t.SliceSize), val.Type())
+		return typeErr(formatSliceString(t.Kind, t.Size), val.Type())
-	}
-	if t.IsArray && val.Len() != t.SliceSize {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), formatSliceString(val.Type().Elem().Kind(), val.Len()))
 	}
 
-	if t.Elem.IsSlice {
+	if t.T == ArrayTy && val.Len() != t.Size {
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), formatSliceString(val.Type().Elem().Kind(), val.Len()))
+	}
+
+	if t.Elem.T == SliceTy {
 		if val.Len() > 0 {
 			return sliceTypeCheck(*t.Elem, val.Index(0))
 		}
-	} else if t.Elem.IsArray {
+	} else if t.Elem.T == ArrayTy {
 		return sliceTypeCheck(*t.Elem, val.Index(0))
 	}
 
 	if elemKind := val.Type().Elem().Kind(); elemKind != t.Elem.Kind {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), val.Type())
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), val.Type())
 	}
 	return nil
 }
@@ -62,20 +63,19 @@ func sliceTypeCheck(t Type, val reflect.Value) error {
 // typeCheck checks that the given reflection value can be assigned to the reflection
 // type in t.
 func typeCheck(t Type, value reflect.Value) error {
-	if t.IsSlice || t.IsArray {
+	if t.T == SliceTy || t.T == ArrayTy {
 		return sliceTypeCheck(t, value)
 	}
 
 	// Check base type validity. Element types will be checked later on.
 	if t.Kind != value.Kind() {
 		return typeErr(t.Kind, value.Kind())
+	} else if t.T == FixedBytesTy && t.Size != value.Len() {
+		return typeErr(t.Type, value.Type())
+	} else {
+		return nil
 	}
-	return nil
-}
 
-// varErr returns a formatted error.
-func varErr(expected, got reflect.Kind) error {
-	return typeErr(expected, got)
 }
 
 // typeErr returns a formatted type casting error.

accounts/abi/event.go

@@ -18,6 +18,7 @@ package abi
 
 import (
 	"fmt"
+	"reflect"
 	"strings"
 
 	"github.com/EthereumCommonwealth/go-callisto/common"
@@ -44,3 +45,93 @@ func (e Event) Id() common.Hash {
 	}
 	return common.BytesToHash(crypto.Keccak256([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")))))
 }
+
+// unpacks an event return tuple into a struct of corresponding go types
+//
+// Unpacking can be done into a struct or a slice/array.
+func (e Event) tupleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	var (
+		value = valueOf.Elem()
+		typ   = value.Type()
+	)
+
+	if value.Kind() != reflect.Struct {
+		return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+	}
+
+	j := 0
+	for i := 0; i < len(e.Inputs); i++ {
+		input := e.Inputs[i]
+		if input.Indexed {
+			// can't read, continue
+			continue
+		} else if input.Type.T == ArrayTy {
+			// need to move this up because they read sequentially
+			j += input.Type.Size
+		}
+		marshalledValue, err := toGoType((i+j)*32, input.Type, output)
+		if err != nil {
+			return err
+		}
+		reflectValue := reflect.ValueOf(marshalledValue)
+
+		switch value.Kind() {
+		case reflect.Struct:
+			for j := 0; j < typ.NumField(); j++ {
+				field := typ.Field(j)
+				// TODO read tags: `abi:"fieldName"`
+				if field.Name == strings.ToUpper(e.Inputs[i].Name[:1])+e.Inputs[i].Name[1:] {
+					if err := set(value.Field(j), reflectValue, e.Inputs[i]); err != nil {
+						return err
+					}
+				}
+			}
+		case reflect.Slice, reflect.Array:
+			if value.Len() < i {
+				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(e.Inputs), value.Len())
+			}
+			v := value.Index(i)
+			if v.Kind() != reflect.Ptr && v.Kind() != reflect.Interface {
+				return fmt.Errorf("abi: cannot unmarshal %v in to %v", v.Type(), reflectValue.Type())
+			}
+			reflectValue := reflect.ValueOf(marshalledValue)
+			if err := set(v.Elem(), reflectValue, e.Inputs[i]); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+		}
+	}
+	return nil
+}
+
+func (e Event) isTupleReturn() bool { return len(e.Inputs) > 1 }
+
+func (e Event) singleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	if e.Inputs[0].Indexed {
+		return fmt.Errorf("abi: attempting to unpack indexed variable into element.")
+	}
+
+	value := valueOf.Elem()
+
+	marshalledValue, err := toGoType(0, e.Inputs[0].Type, output)
+	if err != nil {
+		return err
+	}
+	if err := set(value, reflect.ValueOf(marshalledValue), e.Inputs[0]); err != nil {
+		return err
+	}
+	return nil
+}

accounts/abi/event_test.go

@@ -31,7 +31,7 @@ func TestEventId(t *testing.T) {
 	}{
 		{
 			definition: `[
-			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint" }] },
+			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
 			{ "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }
 			]`,
 			expectations: map[string]common.Hash{

accounts/abi/method.go

@@ -77,6 +77,85 @@ func (method Method) pack(args ...interface{}) ([]byte, error) {
 	return ret, nil
 }
 
+// unpacks a method return tuple into a struct of corresponding go types
+//
+// Unpacking can be done into a struct or a slice/array.
+func (method Method) tupleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	var (
+		value = valueOf.Elem()
+		typ   = value.Type()
+	)
+
+	j := 0
+	for i := 0; i < len(method.Outputs); i++ {
+		toUnpack := method.Outputs[i]
+		if toUnpack.Type.T == ArrayTy {
+			// need to move this up because they read sequentially
+			j += toUnpack.Type.Size
+		}
+		marshalledValue, err := toGoType((i+j)*32, toUnpack.Type, output)
+		if err != nil {
+			return err
+		}
+		reflectValue := reflect.ValueOf(marshalledValue)
+
+		switch value.Kind() {
+		case reflect.Struct:
+			for j := 0; j < typ.NumField(); j++ {
+				field := typ.Field(j)
+				// TODO read tags: `abi:"fieldName"`
+				if field.Name == strings.ToUpper(method.Outputs[i].Name[:1])+method.Outputs[i].Name[1:] {
+					if err := set(value.Field(j), reflectValue, method.Outputs[i]); err != nil {
+						return err
+					}
+				}
+			}
+		case reflect.Slice, reflect.Array:
+			if value.Len() < i {
+				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(method.Outputs), value.Len())
+			}
+			v := value.Index(i)
+			if v.Kind() != reflect.Ptr && v.Kind() != reflect.Interface {
+				return fmt.Errorf("abi: cannot unmarshal %v in to %v", v.Type(), reflectValue.Type())
+			}
+			reflectValue := reflect.ValueOf(marshalledValue)
+			if err := set(v.Elem(), reflectValue, method.Outputs[i]); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
+		}
+	}
+	return nil
+}
+
+func (method Method) isTupleReturn() bool { return len(method.Outputs) > 1 }
+
+func (method Method) singleUnpack(v interface{}, output []byte) error {
+	// make sure the passed value is a pointer
+	valueOf := reflect.ValueOf(v)
+	if reflect.Ptr != valueOf.Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+
+	value := valueOf.Elem()
+
+	marshalledValue, err := toGoType(0, method.Outputs[0].Type, output)
+	if err != nil {
+		return err
+	}
+	if err := set(value, reflect.ValueOf(marshalledValue), method.Outputs[0]); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Sig returns the methods string signature according to the ABI spec.
 //
 // Example

accounts/abi/numbers.go

@@ -25,36 +25,23 @@ import (
 )
 
 var (
-	big_t     = reflect.TypeOf(big.Int{})
+	big_t      = reflect.TypeOf(&big.Int{})
-	ubig_t    = reflect.TypeOf(big.Int{})
+	derefbig_t = reflect.TypeOf(big.Int{})
-	byte_t    = reflect.TypeOf(byte(0))
+	uint8_t    = reflect.TypeOf(uint8(0))
-	byte_ts   = reflect.TypeOf([]byte(nil))
+	uint16_t   = reflect.TypeOf(uint16(0))
-	uint_t    = reflect.TypeOf(uint(0))
+	uint32_t   = reflect.TypeOf(uint32(0))
-	uint8_t   = reflect.TypeOf(uint8(0))
+	uint64_t   = reflect.TypeOf(uint64(0))
-	uint16_t  = reflect.TypeOf(uint16(0))
+	int_t      = reflect.TypeOf(int(0))
-	uint32_t  = reflect.TypeOf(uint32(0))
+	int8_t     = reflect.TypeOf(int8(0))
-	uint64_t  = reflect.TypeOf(uint64(0))
+	int16_t    = reflect.TypeOf(int16(0))
-	int_t     = reflect.TypeOf(int(0))
+	int32_t    = reflect.TypeOf(int32(0))
-	int8_t    = reflect.TypeOf(int8(0))
+	int64_t    = reflect.TypeOf(int64(0))
-	int16_t   = reflect.TypeOf(int16(0))
+	address_t  = reflect.TypeOf(common.Address{})
-	int32_t   = reflect.TypeOf(int32(0))
+	int_ts     = reflect.TypeOf([]int(nil))
-	int64_t   = reflect.TypeOf(int64(0))
+	int8_ts    = reflect.TypeOf([]int8(nil))
-	hash_t    = reflect.TypeOf(common.Hash{})
+	int16_ts   = reflect.TypeOf([]int16(nil))
-	address_t = reflect.TypeOf(common.Address{})
+	int32_ts   = reflect.TypeOf([]int32(nil))
-
+	int64_ts   = reflect.TypeOf([]int64(nil))
-	uint_ts   = reflect.TypeOf([]uint(nil))
-	uint8_ts  = reflect.TypeOf([]uint8(nil))
-	uint16_ts = reflect.TypeOf([]uint16(nil))
-	uint32_ts = reflect.TypeOf([]uint32(nil))
-	uint64_ts = reflect.TypeOf([]uint64(nil))
-	ubig_ts   = reflect.TypeOf([]*big.Int(nil))
-
-	int_ts   = reflect.TypeOf([]int(nil))
-	int8_ts  = reflect.TypeOf([]int8(nil))
-	int16_ts = reflect.TypeOf([]int16(nil))
-	int32_ts = reflect.TypeOf([]int32(nil))
-	int64_ts = reflect.TypeOf([]int64(nil))
-	big_ts   = reflect.TypeOf([]*big.Int(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.

accounts/abi/pack.go

@@ -61,8 +61,9 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 			reflectValue = mustArrayToByteSlice(reflectValue)
 		}
 		return common.RightPadBytes(reflectValue.Bytes(), 32)
+	default:
+		panic("abi: fatal error")
 	}
-	panic("abi: fatal error")
 }
 
 // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation
@@ -74,6 +75,8 @@ func packNum(value reflect.Value) []byte {
 		return U256(big.NewInt(value.Int()))
 	case reflect.Ptr:
 		return U256(value.Interface().(*big.Int))
+	default:
+		panic("abi: fatal error")
 	}
-	return nil
+
 }

accounts/abi/pack_test.go

@@ -322,12 +322,12 @@ func TestPack(t *testing.T) {
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
-			t.Fatal("unexpected parse error:", err)
+			t.Fatalf("%v failed. Unexpected parse error: %v", i, err)
 		}
 
 		output, err := typ.pack(reflect.ValueOf(test.input))
 		if err != nil {
-			t.Fatal("unexpected pack error:", err)
+			t.Fatalf("%v failed. Unexpected pack error: %v", i, err)
 		}
 
 		if !bytes.Equal(output, test.output) {
@@ -435,7 +435,4 @@ func TestPackNumber(t *testing.T) {
 			t.Errorf("test %d: pack mismatch: have %x, want %x", i, packed, tt.packed)
 		}
 	}
-	if packed := packNum(reflect.ValueOf("string")); packed != nil {
-		t.Errorf("expected 'string' to pack to nil. got %x instead", packed)
-	}
 }

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != big_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
 		return indirect(v.Elem())
 	}
 	return v
@@ -73,15 +73,9 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 func set(dst, src reflect.Value, output Argument) error {
 	dstType := dst.Type()
 	srcType := src.Type()
-
 	switch {
-	case dstType.AssignableTo(src.Type()):
+	case dstType.AssignableTo(srcType):
 		dst.Set(src)
-	case dstType.Kind() == reflect.Array && srcType.Kind() == reflect.Slice:
-		if dst.Len() < output.Type.SliceSize {
-			return fmt.Errorf("abi: cannot unmarshal src (len=%d) in to dst (len=%d)", output.Type.SliceSize, dst.Len())
-		}
-		reflect.Copy(dst, src)
 	case dstType.Kind() == reflect.Interface:
 		dst.Set(src)
 	case dstType.Kind() == reflect.Ptr:

accounts/abi/type.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 )
 
 const (
@@ -29,6 +30,7 @@ const (
 	BoolTy
 	StringTy
 	SliceTy
+	ArrayTy
 	AddressTy
 	FixedBytesTy
 	BytesTy
@@ -39,9 +41,6 @@ const (
 
 // Type is the reflection of the supported argument type
 type Type struct {
-	IsSlice, IsArray bool
-	SliceSize        int
-
 	Elem *Type
 
 	Kind reflect.Kind
@@ -53,118 +52,116 @@ type Type struct {
 }
 
 var (
-	// fullTypeRegex parses the abi types
-	//
-	// Types can be in the format of:
-	//
-	// 	Input  = Type [ "[" [ Number ] "]" ] Name .
-	// 	Type   = [ "u" ] "int" [ Number ] [ x ] [ Number ].
-	//
-	// Examples:
-	//
-	//      string     int       uint       fixed
-	//      string32   int8      uint8      uint[]
-	//      address    int256    uint256    fixed128x128[2]
-	fullTypeRegex = regexp.MustCompile(`([a-zA-Z0-9]+)(\[([0-9]*)\])?`)
 	// typeRegex parses the abi sub types
 	typeRegex = regexp.MustCompile("([a-zA-Z]+)(([0-9]+)(x([0-9]+))?)?")
 )
 
 // NewType creates a new reflection type of abi type given in t.
 func NewType(t string) (typ Type, err error) {
-	res := fullTypeRegex.FindAllStringSubmatch(t, -1)[0]
+	// check that array brackets are equal if they exist
-	// check if type is slice and parse type.
+	if strings.Count(t, "[") != strings.Count(t, "]") {
-	switch {
+		return Type{}, fmt.Errorf("invalid arg type in abi")
-	case res[3] != "":
-		// err is ignored. Already checked for number through the regexp
-		typ.SliceSize, _ = strconv.Atoi(res[3])
-		typ.IsArray = true
-	case res[2] != "":
-		typ.IsSlice, typ.SliceSize = true, -1
-	case res[0] == "":
-		return Type{}, fmt.Errorf("abi: type parse error: %s", t)
 	}
-	if typ.IsArray || typ.IsSlice {
+
-		sliceType, err := NewType(res[1])
+	typ.stringKind = t
+
+	// if there are brackets, get ready to go into slice/array mode and
+	// recursively create the type
+	if strings.Count(t, "[") != 0 {
+		i := strings.LastIndex(t, "[")
+		// recursively embed the type
+		embeddedType, err := NewType(t[:i])
 		if err != nil {
 			return Type{}, err
 		}
-		typ.Elem = &sliceType
+		// grab the last cell and create a type from there
-		typ.stringKind = sliceType.stringKind + t[len(res[1]):]
+		sliced := t[i:]
-		// Although we know that this is an array, we cannot return
+		// grab the slice size with regexp
-		// as we don't know the type of the element, however, if it
+		re := regexp.MustCompile("[0-9]+")
-		// is still an array, then don't determine the type.
+		intz := re.FindAllString(sliced, -1)
-		if typ.Elem.IsArray || typ.Elem.IsSlice {
-			return typ, nil
-		}
-	}
 
-	// parse the type and size of the abi-type.
+		if len(intz) == 0 {
-	parsedType := typeRegex.FindAllStringSubmatch(res[1], -1)[0]
+			// is a slice
-	// varSize is the size of the variable
+			typ.T = SliceTy
-	var varSize int
+			typ.Kind = reflect.Slice
-	if len(parsedType[3]) > 0 {
+			typ.Elem = &embeddedType
-		var err error
+			typ.Type = reflect.SliceOf(embeddedType.Type)
-		varSize, err = strconv.Atoi(parsedType[2])
+		} else if len(intz) == 1 {
-		if err != nil {
+			// is a array
-			return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
+			typ.T = ArrayTy
-		}
+			typ.Kind = reflect.Array
-	}
+			typ.Elem = &embeddedType
-	// varType is the parsed abi type
+			typ.Size, err = strconv.Atoi(intz[0])
-	varType := parsedType[1]
+			if err != nil {
-	// substitute canonical integer
+				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
-	if varSize == 0 && (varType == "int" || varType == "uint") {
+			}
-		varSize = 256
+			typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type)
-		t += "256"
-	}
-
-	// only set stringKind if not array or slice, as for those,
-	// the correct string type has been set
-	if !(typ.IsArray || typ.IsSlice) {
-		typ.stringKind = t
-	}
-
-	switch varType {
-	case "int":
-		typ.Kind, typ.Type = reflectIntKindAndType(false, varSize)
-		typ.Size = varSize
-		typ.T = IntTy
-	case "uint":
-		typ.Kind, typ.Type = reflectIntKindAndType(true, varSize)
-		typ.Size = varSize
-		typ.T = UintTy
-	case "bool":
-		typ.Kind = reflect.Bool
-		typ.T = BoolTy
-	case "address":
-		typ.Kind = reflect.Array
-		typ.Type = address_t
-		typ.Size = 20
-		typ.T = AddressTy
-	case "string":
-		typ.Kind = reflect.String
-		typ.Size = -1
-		typ.T = StringTy
-	case "bytes":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
-		if varSize == 0 {
-			typ.IsSlice = true
-			typ.T = BytesTy
-			typ.SliceSize = -1
 		} else {
-			typ.IsArray = true
+			return Type{}, fmt.Errorf("invalid formatting of array type")
-			typ.T = FixedBytesTy
+		}
-			typ.SliceSize = varSize
+		return typ, err
+	} else {
+		// parse the type and size of the abi-type.
+		parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+		// varSize is the size of the variable
+		var varSize int
+		if len(parsedType[3]) > 0 {
+			var err error
+			varSize, err = strconv.Atoi(parsedType[2])
+			if err != nil {
+				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
+			}
+		} else {
+			if parsedType[0] == "uint" || parsedType[0] == "int" {
+				// this should fail because it means that there's something wrong with
+				// the abi type (the compiler should always format it to the size...always)
+				return Type{}, fmt.Errorf("unsupported arg type: %s", t)
+			}
+		}
+		// varType is the parsed abi type
+		varType := parsedType[1]
+
+		switch varType {
+		case "int":
+			typ.Kind, typ.Type = reflectIntKindAndType(false, varSize)
+			typ.Size = varSize
+			typ.T = IntTy
+		case "uint":
+			typ.Kind, typ.Type = reflectIntKindAndType(true, varSize)
+			typ.Size = varSize
+			typ.T = UintTy
+		case "bool":
+			typ.Kind = reflect.Bool
+			typ.T = BoolTy
+			typ.Type = reflect.TypeOf(bool(false))
+		case "address":
+			typ.Kind = reflect.Array
+			typ.Type = address_t
+			typ.Size = 20
+			typ.T = AddressTy
+		case "string":
+			typ.Kind = reflect.String
+			typ.Type = reflect.TypeOf("")
+			typ.T = StringTy
+		case "bytes":
+			if varSize == 0 {
+				typ.T = BytesTy
+				typ.Kind = reflect.Slice
+				typ.Type = reflect.SliceOf(reflect.TypeOf(byte(0)))
+			} else {
+				typ.T = FixedBytesTy
+				typ.Kind = reflect.Array
+				typ.Size = varSize
+				typ.Type = reflect.ArrayOf(varSize, reflect.TypeOf(byte(0)))
+			}
+		case "function":
+			typ.Kind = reflect.Array
+			typ.T = FunctionTy
+			typ.Size = 24
+			typ.Type = reflect.ArrayOf(24, reflect.TypeOf(byte(0)))
+		default:
+			return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 		}
-	case "function":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
-		typ.IsArray = true
-		typ.T = FunctionTy
-		typ.SliceSize = 24
-	default:
-		return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 	}
 
 	return
@@ -183,7 +180,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if (t.IsSlice || t.IsArray) && t.T != BytesTy && t.T != FixedBytesTy && t.T != FunctionTy {
+	if t.T == SliceTy || t.T == ArrayTy {
 		var packed []byte
 
 		for i := 0; i < v.Len(); i++ {
@@ -193,18 +190,17 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 			}
 			packed = append(packed, val...)
 		}
-		if t.IsSlice {
+		if t.T == SliceTy {
 			return packBytesSlice(packed, v.Len()), nil
-		} else if t.IsArray {
+		} else if t.T == ArrayTy {
 			return packed, nil
 		}
 	}
-
 	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
 // prefixing.
 func (t Type) requiresLengthPrefix() bool {
-	return t.T != FixedBytesTy && (t.T == StringTy || t.T == BytesTy || t.IsSlice)
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }

accounts/abi/type_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/EthereumCommonwealth/go-callisto/common"
+	"github.com/davecgh/go-spew/spew"
 )
 
 // typeWithoutStringer is a alias for the Type type which simply doesn't implement
@@ -34,51 +35,58 @@ func TestTypeRegexp(t *testing.T) {
 		blob string
 		kind Type
 	}{
-		{"bool", Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}},
+		{"bool", Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}},
-		{"bool[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[]"}},
+		{"bool[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}},
-		{"bool[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool[2][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}},
+		{"bool[][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}},
+		{"bool[][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}},
+		{"bool[2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}},
+		{"bool[2][][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}},
+		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
+		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
+		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
 		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
 		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
 		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
 		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
 		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
 		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
 		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
 		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
 		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
 		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
-		{"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}},
+		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
-		{"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
+		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
-		{"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}},
+		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
-		{"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
+		{"bytes32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
-		{"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
+		{"bytes32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
-		{"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}},
+		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
-		{"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}},
+		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
-		{"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}},
+		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
 		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
-
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -87,13 +95,14 @@ func TestTypeRegexp(t *testing.T) {
 		// {"fixed128x128[]", Type{}},
 		// {"fixed128x128[2]", Type{}},
 	}
-	for i, tt := range tests {
+
+	for _, tt := range tests {
 		typ, err := NewType(tt.blob)
 		if err != nil {
-			t.Errorf("type %d: failed to parse type string: %v", i, err)
+			t.Errorf("type %q: failed to parse type string: %v", tt.blob, err)
 		}
 		if !reflect.DeepEqual(typ, tt.kind) {
-			t.Errorf("type %d: parsed type mismatch:\n  have %+v\n  want %+v", i, typeWithoutStringer(typ), typeWithoutStringer(tt.kind))
+			t.Errorf("type %q: parsed type mismatch:\nGOT %s\nWANT %s ", tt.blob, spew.Sdump(typeWithoutStringer(typ)), spew.Sdump(typeWithoutStringer(tt.kind)))
 		}
 	}
 }
@@ -104,15 +113,90 @@ func TestTypeCheck(t *testing.T) {
 		input interface{}
 		err   string
 	}{
-		{"uint", big.NewInt(1), ""},
+		{"uint", big.NewInt(1), "unsupported arg type: uint"},
-		{"int", big.NewInt(1), ""},
+		{"int", big.NewInt(1), "unsupported arg type: int"},
-		{"uint30", big.NewInt(1), ""},
+		{"uint256", big.NewInt(1), ""},
+		{"uint256[][3][]", [][3][]*big.Int{{{}}}, ""},
+		{"uint256[][][3]", [3][][]*big.Int{{{}}}, ""},
+		{"uint256[3][][]", [][][3]*big.Int{{{}}}, ""},
+		{"uint256[3][3][3]", [3][3][3]*big.Int{{{}}}, ""},
+		{"uint8[][]", [][]uint8{}, ""},
+		{"int256", big.NewInt(1), ""},
+		{"uint8", uint8(1), ""},
+		{"uint16", uint16(1), ""},
+		{"uint32", uint32(1), ""},
+		{"uint64", uint64(1), ""},
+		{"int8", int8(1), ""},
+		{"int16", int16(1), ""},
+		{"int32", int32(1), ""},
+		{"int64", int64(1), ""},
+		{"uint24", big.NewInt(1), ""},
+		{"uint40", big.NewInt(1), ""},
+		{"uint48", big.NewInt(1), ""},
+		{"uint56", big.NewInt(1), ""},
+		{"uint72", big.NewInt(1), ""},
+		{"uint80", big.NewInt(1), ""},
+		{"uint88", big.NewInt(1), ""},
+		{"uint96", big.NewInt(1), ""},
+		{"uint104", big.NewInt(1), ""},
+		{"uint112", big.NewInt(1), ""},
+		{"uint120", big.NewInt(1), ""},
+		{"uint128", big.NewInt(1), ""},
+		{"uint136", big.NewInt(1), ""},
+		{"uint144", big.NewInt(1), ""},
+		{"uint152", big.NewInt(1), ""},
+		{"uint160", big.NewInt(1), ""},
+		{"uint168", big.NewInt(1), ""},
+		{"uint176", big.NewInt(1), ""},
+		{"uint184", big.NewInt(1), ""},
+		{"uint192", big.NewInt(1), ""},
+		{"uint200", big.NewInt(1), ""},
+		{"uint208", big.NewInt(1), ""},
+		{"uint216", big.NewInt(1), ""},
+		{"uint224", big.NewInt(1), ""},
+		{"uint232", big.NewInt(1), ""},
+		{"uint240", big.NewInt(1), ""},
+		{"uint248", big.NewInt(1), ""},
+		{"int24", big.NewInt(1), ""},
+		{"int40", big.NewInt(1), ""},
+		{"int48", big.NewInt(1), ""},
+		{"int56", big.NewInt(1), ""},
+		{"int72", big.NewInt(1), ""},
+		{"int80", big.NewInt(1), ""},
+		{"int88", big.NewInt(1), ""},
+		{"int96", big.NewInt(1), ""},
+		{"int104", big.NewInt(1), ""},
+		{"int112", big.NewInt(1), ""},
+		{"int120", big.NewInt(1), ""},
+		{"int128", big.NewInt(1), ""},
+		{"int136", big.NewInt(1), ""},
+		{"int144", big.NewInt(1), ""},
+		{"int152", big.NewInt(1), ""},
+		{"int160", big.NewInt(1), ""},
+		{"int168", big.NewInt(1), ""},
+		{"int176", big.NewInt(1), ""},
+		{"int184", big.NewInt(1), ""},
+		{"int192", big.NewInt(1), ""},
+		{"int200", big.NewInt(1), ""},
+		{"int208", big.NewInt(1), ""},
+		{"int216", big.NewInt(1), ""},
+		{"int224", big.NewInt(1), ""},
+		{"int232", big.NewInt(1), ""},
+		{"int240", big.NewInt(1), ""},
+		{"int248", big.NewInt(1), ""},
 		{"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"},
+		{"uint8", uint16(1), "abi: cannot use uint16 as type uint8 as argument"},
+		{"uint8", uint32(1), "abi: cannot use uint32 as type uint8 as argument"},
+		{"uint8", uint64(1), "abi: cannot use uint64 as type uint8 as argument"},
+		{"uint8", int8(1), "abi: cannot use int8 as type uint8 as argument"},
+		{"uint8", int16(1), "abi: cannot use int16 as type uint8 as argument"},
+		{"uint8", int32(1), "abi: cannot use int32 as type uint8 as argument"},
+		{"uint8", int64(1), "abi: cannot use int64 as type uint8 as argument"},
 		{"uint16", uint16(1), ""},
 		{"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"},
 		{"uint16[]", []uint16{1, 2, 3}, ""},
 		{"uint16[]", [3]uint16{1, 2, 3}, ""},
-		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type []uint16 as argument"},
+		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"},
 		{"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"},
 		{"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
 		{"uint16[3]", []uint16{1, 2, 3}, ""},
@@ -122,20 +206,61 @@ func TestTypeCheck(t *testing.T) {
 		{"address[1]", [1]common.Address{{1}}, ""},
 		{"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
 		{"bytes32", [32]byte{}, ""},
+		{"bytes31", [31]byte{}, ""},
+		{"bytes30", [30]byte{}, ""},
+		{"bytes29", [29]byte{}, ""},
+		{"bytes28", [28]byte{}, ""},
+		{"bytes27", [27]byte{}, ""},
+		{"bytes26", [26]byte{}, ""},
+		{"bytes25", [25]byte{}, ""},
+		{"bytes24", [24]byte{}, ""},
+		{"bytes23", [23]byte{}, ""},
+		{"bytes22", [22]byte{}, ""},
+		{"bytes21", [21]byte{}, ""},
+		{"bytes20", [20]byte{}, ""},
+		{"bytes19", [19]byte{}, ""},
+		{"bytes18", [18]byte{}, ""},
+		{"bytes17", [17]byte{}, ""},
+		{"bytes16", [16]byte{}, ""},
+		{"bytes15", [15]byte{}, ""},
+		{"bytes14", [14]byte{}, ""},
+		{"bytes13", [13]byte{}, ""},
+		{"bytes12", [12]byte{}, ""},
+		{"bytes11", [11]byte{}, ""},
+		{"bytes10", [10]byte{}, ""},
+		{"bytes9", [9]byte{}, ""},
+		{"bytes8", [8]byte{}, ""},
+		{"bytes7", [7]byte{}, ""},
+		{"bytes6", [6]byte{}, ""},
+		{"bytes5", [5]byte{}, ""},
+		{"bytes4", [4]byte{}, ""},
+		{"bytes3", [3]byte{}, ""},
+		{"bytes2", [2]byte{}, ""},
+		{"bytes1", [1]byte{}, ""},
 		{"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
 		{"bytes32", common.Hash{1}, ""},
-		{"bytes31", [31]byte{}, ""},
+		{"bytes31", common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"},
 		{"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"},
 		{"bytes", []byte{0, 1}, ""},
-		{"bytes", [2]byte{0, 1}, ""},
+		{"bytes", [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
-		{"bytes", common.Hash{1}, ""},
+		{"bytes", common.Hash{1}, "abi: cannot use array as type slice as argument"},
 		{"string", "hello world", ""},
+		{"string", string(""), ""},
+		{"string", []byte{}, "abi: cannot use slice as type string as argument"},
 		{"bytes32[]", [][32]byte{{}}, ""},
 		{"function", [24]byte{}, ""},
+		{"bytes20", common.Address{}, ""},
+		{"address", [20]byte{}, ""},
+		{"address", common.Address{}, ""},
 	} {
 		typ, err := NewType(test.typ)
-		if err != nil {
+		if err != nil && len(test.err) == 0 {
 			t.Fatal("unexpected parse error:", err)
+		} else if err != nil && len(test.err) != 0 {
+			if err.Error() != test.err {
+				t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err)
+			}
+			continue
 		}
 
 		err = typeCheck(typ, reflect.ValueOf(test.input))

accounts/abi/unpack.go

@@ -25,122 +25,20 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/common"
 )
 
-// toGoSliceType parses the input and casts it to the proper slice defined by the ABI
+// unpacker is a utility interface that enables us to have
-// argument in T.
+// abstraction between events and methods and also to properly
-func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
+// "unpack" them; e.g. events use Inputs, methods use Outputs.
-	index := i * 32
+type unpacker interface {
-	// The slice must, at very least be large enough for the index+32 which is exactly the size required
+	tupleUnpack(v interface{}, output []byte) error
-	// for the [offset in output, size of offset].
+	singleUnpack(v interface{}, output []byte) error
-	if index+32 > len(output) {
+	isTupleReturn() bool
-		return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), index+32)
-	}
-	elem := t.Type.Elem
-
-	// first we need to create a slice of the type
-	var refSlice reflect.Value
-	switch elem.T {
-	case IntTy, UintTy, BoolTy:
-		// create a new reference slice matching the element type
-		switch t.Type.Kind {
-		case reflect.Bool:
-			refSlice = reflect.ValueOf([]bool(nil))
-		case reflect.Uint8:
-			refSlice = reflect.ValueOf([]uint8(nil))
-		case reflect.Uint16:
-			refSlice = reflect.ValueOf([]uint16(nil))
-		case reflect.Uint32:
-			refSlice = reflect.ValueOf([]uint32(nil))
-		case reflect.Uint64:
-			refSlice = reflect.ValueOf([]uint64(nil))
-		case reflect.Int8:
-			refSlice = reflect.ValueOf([]int8(nil))
-		case reflect.Int16:
-			refSlice = reflect.ValueOf([]int16(nil))
-		case reflect.Int32:
-			refSlice = reflect.ValueOf([]int32(nil))
-		case reflect.Int64:
-			refSlice = reflect.ValueOf([]int64(nil))
-		default:
-			refSlice = reflect.ValueOf([]*big.Int(nil))
-		}
-	case AddressTy: // address must be of slice Address
-		refSlice = reflect.ValueOf([]common.Address(nil))
-	case HashTy: // hash must be of slice hash
-		refSlice = reflect.ValueOf([]common.Hash(nil))
-	case FixedBytesTy:
-		refSlice = reflect.ValueOf([][]byte(nil))
-	default: // no other types are supported
-		return nil, fmt.Errorf("abi: unsupported slice type %v", elem.T)
-	}
-
-	var slice []byte
-	var size int
-	var offset int
-	if t.Type.IsSlice {
-		// get the offset which determines the start of this array ...
-		offset = int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
-		}
-
-		slice = output[offset:]
-		// ... starting with the size of the array in elements ...
-		size = int(binary.BigEndian.Uint64(slice[24:32]))
-		slice = slice[32:]
-		// ... and make sure that we've at the very least the amount of bytes
-		// available in the buffer.
-		if size*32 > len(slice) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), offset+32+size*32)
-		}
-
-		// reslice to match the required size
-		slice = slice[:size*32]
-	} else if t.Type.IsArray {
-		//get the number of elements in the array
-		size = t.Type.SliceSize
-
-		//check to make sure array size matches up
-		if index+32*size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), index+32*size)
-		}
-		//slice is there for a fixed amount of times
-		slice = output[index : index+size*32]
-	}
-
-	for i := 0; i < size; i++ {
-		var (
-			inter        interface{}             // interface type
-			returnOutput = slice[i*32 : i*32+32] // the return output
-			err          error
-		)
-		// set inter to the correct type (cast)
-		switch elem.T {
-		case IntTy, UintTy:
-			inter = readInteger(t.Type.Kind, returnOutput)
-		case BoolTy:
-			inter, err = readBool(returnOutput)
-			if err != nil {
-				return nil, err
-			}
-		case AddressTy:
-			inter = common.BytesToAddress(returnOutput)
-		case HashTy:
-			inter = common.BytesToHash(returnOutput)
-		case FixedBytesTy:
-			inter = returnOutput
-		}
-		// append the item to our reflect slice
-		refSlice = reflect.Append(refSlice, reflect.ValueOf(inter))
-	}
-
-	// return the interface
-	return refSlice.Interface(), nil
 }
 
+// reads the integer based on its kind
 func readInteger(kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
-		return uint8(b[len(b)-1])
+		return b[len(b)-1]
 	case reflect.Uint16:
 		return binary.BigEndian.Uint16(b[len(b)-2:])
 	case reflect.Uint32:
@@ -160,13 +58,10 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	}
 }
 
+// reads a bool
 func readBool(word []byte) (bool, error) {
-	if len(word) != 32 {
+	for _, b := range word[:31] {
-		return false, fmt.Errorf("abi: fatal error: incorrect word length")
+		if b != 0 {
-	}
-
-	for i, b := range word {
-		if b != 0 && i != 31 {
 			return false, errBadBool
 		}
 	}
@@ -178,58 +73,144 @@ func readBool(word []byte) (bool, error) {
 	default:
 		return false, errBadBool
 	}
+}
+
+// A function type is simply the address with the function selection signature at the end.
+// This enforces that standard by always presenting it as a 24-array (address + sig = 24 bytes)
+func readFunctionType(t Type, word []byte) (funcTy [24]byte, err error) {
+	if t.T != FunctionTy {
+		return [24]byte{}, fmt.Errorf("abi: invalid type in call to make function type byte array.")
+	}
+	if garbage := binary.BigEndian.Uint64(word[24:32]); garbage != 0 {
+		err = fmt.Errorf("abi: got improperly encoded function type, got %v", word)
+	} else {
+		copy(funcTy[:], word[0:24])
+	}
+	return
+}
+
+// through reflection, creates a fixed array to be read from
+func readFixedBytes(t Type, word []byte) (interface{}, error) {
+	if t.T != FixedBytesTy {
+		return nil, fmt.Errorf("abi: invalid type in call to make fixed byte array.")
+	}
+	// convert
+	array := reflect.New(t.Type).Elem()
+
+	reflect.Copy(array, reflect.ValueOf(word[0:t.Size]))
+	return array.Interface(), nil
 
 }
 
-// toGoType parses the input and casts it to the proper type defined by the ABI
+// iteratively unpack elements
-// argument in T.
+func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
-func toGoType(i int, t Argument, output []byte) (interface{}, error) {
+	if start+32*size > len(output) {
-	// we need to treat slices differently
+		return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
-	if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy {
-		return toGoSlice(i, t, output)
 	}
 
-	index := i * 32
+	// this value will become our slice or our array, depending on the type
+	var refSlice reflect.Value
+	slice := output[start : start+size*32]
+
+	if t.T == SliceTy {
+		// declare our slice
+		refSlice = reflect.MakeSlice(t.Type, size, size)
+	} else if t.T == ArrayTy {
+		// declare our array
+		refSlice = reflect.New(t.Type).Elem()
+	} else {
+		return nil, fmt.Errorf("abi: invalid type in array/slice unpacking stage")
+	}
+
+	for i, j := start, 0; j*32 < len(slice); i, j = i+32, j+1 {
+		// this corrects the arrangement so that we get all the underlying array values
+		if t.Elem.T == ArrayTy && j != 0 {
+			i = start + t.Elem.Size*32*j
+		}
+		inter, err := toGoType(i, *t.Elem, output)
+		if err != nil {
+			return nil, err
+		}
+		// append the item to our reflect slice
+		refSlice.Index(j).Set(reflect.ValueOf(inter))
+	}
+
+	// return the interface
+	return refSlice.Interface(), nil
+}
+
+// toGoType parses the output bytes and recursively assigns the value of these bytes
+// into a go type with accordance with the ABI spec.
+func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	if index+32 > len(output) {
 		return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32)
 	}
 
-	// Parse the given index output and check whether we need to read
+	var (
-	// a different offset and length based on the type (i.e. string, bytes)
+		returnOutput []byte
-	var returnOutput []byte
+		begin, end   int
-	switch t.Type.T {
+		err          error
-	case StringTy, BytesTy: // variable arrays are written at the end of the return bytes
+	)
-		// parse offset from which we should start reading
-		offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32)
-		}
-		// parse the size up until we should be reading
-		size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
-		if offset+32+size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size)
-		}
 
-		// get the bytes for this return value
+	// if we require a length prefix, find the beginning word and size returned.
-		returnOutput = output[offset+32 : offset+32+size]
+	if t.requiresLengthPrefix() {
-	default:
+		begin, end, err = lengthPrefixPointsTo(index, output)
+		if err != nil {
+			return nil, err
+		}
+	} else {
 		returnOutput = output[index : index+32]
 	}
 
-	// convert the bytes to whatever is specified by the ABI.
+	switch t.T {
-	switch t.Type.T {
+	case SliceTy:
+		return forEachUnpack(t, output, begin, end)
+	case ArrayTy:
+		return forEachUnpack(t, output, index, t.Size)
+	case StringTy: // variable arrays are written at the end of the return bytes
+		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Type.Kind, returnOutput), nil
+		return readInteger(t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:
 		return common.BytesToHash(returnOutput), nil
-	case BytesTy, FixedBytesTy, FunctionTy:
+	case BytesTy:
-		return returnOutput, nil
+		return output[begin : begin+end], nil
-	case StringTy:
+	case FixedBytesTy:
-		return string(returnOutput), nil
+		return readFixedBytes(t, returnOutput)
+	case FunctionTy:
+		return readFunctionType(t, returnOutput)
+	default:
+		return nil, fmt.Errorf("abi: unknown type %v", t.T)
+	}
+}
+
+// interprets a 32 byte slice as an offset and then determines which indice to look to decode the type.
+func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) {
+	offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
+	if offset+32 > len(output) {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
+	}
+	length = int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
+	if offset+32+length > len(output) {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length)
+	}
+	start = offset + 32
+
+	//fmt.Printf("LENGTH PREFIX INFO: \nsize: %v\noffset: %v\nstart: %v\n", length, offset, start)
+	return
+}
+
+// checks for proper formatting of byte output
+func bytesAreProper(output []byte) error {
+	if len(output) == 0 {
+		return fmt.Errorf("abi: unmarshalling empty output")
+	} else if len(output)%32 != 0 {
+		return fmt.Errorf("abi: improperly formatted output")
+	} else {
+		return nil
 	}
-	return nil, fmt.Errorf("abi: unknown type %v", t.Type.T)
 }

accounts/abi/unpack_test.go

@@ -18,6 +18,7 @@ package abi
 
 import (
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"math/big"
 	"reflect"
@@ -27,260 +28,258 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/common"
 )
 
-func TestSimpleMethodUnpack(t *testing.T) {
+type unpackTest struct {
-	for i, test := range []struct {
+	def  string      // ABI definition JSON
-		def              string      // definition of the **output** ABI params
+	enc  string      // evm return data
-		marshalledOutput []byte      // evm return data
+	want interface{} // the expected output
-		expectedOut      interface{} // the expected output
+	err  string      // empty or error if expected
-		outVar           string      // the output variable (e.g. uint32, *big.Int, etc)
+}
-		err              string      // empty or error if expected
-	}{
-		{
-			`[ { "type": "bool" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			bool(true),
-			"bool",
-			"",
-		},
-		{
-			`[ { "type": "uint32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			uint32(1),
-			"uint32",
-			"",
-		},
-		{
-			`[ { "type": "uint32" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"uint16",
-			"abi: cannot unmarshal uint32 in to uint16",
-		},
-		{
-			`[ { "type": "uint17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"uint16",
-			"abi: cannot unmarshal *big.Int in to uint16",
-		},
-		{
-			`[ { "type": "uint17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			big.NewInt(1),
-			"*big.Int",
-			"",
-		},
 
-		{
+func (test unpackTest) checkError(err error) error {
-			`[ { "type": "int32" } ]`,
+	if err != nil {
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
+		if len(test.err) == 0 {
-			int32(1),
+			return fmt.Errorf("expected no err but got: %v", err)
-			"int32",
+		} else if err.Error() != test.err {
-			"",
+			return fmt.Errorf("expected err: '%v' got err: %q", test.err, err)
-		},
+		}
-		{
+	} else if len(test.err) > 0 {
-			`[ { "type": "int32" } ]`,
+		return fmt.Errorf("expected err: %v but got none", test.err)
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
+	}
-			nil,
+	return nil
-			"int16",
+}
-			"abi: cannot unmarshal int32 in to int16",
-		},
-		{
-			`[ { "type": "int17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			nil,
-			"int16",
-			"abi: cannot unmarshal *big.Int in to int16",
-		},
-		{
-			`[ { "type": "int17" } ]`,
-			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"),
-			big.NewInt(1),
-			"*big.Int",
-			"",
-		},
 
-		{
+var unpackTests = []unpackTest{
-			`[ { "type": "address" } ]`,
+	{
-			common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000"),
+		def:  `[{ "type": "bool" }]`,
-			common.Address{1},
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-			"address",
+		want: true,
-			"",
+	},
-		},
+	{
-		{
+		def:  `[{"type": "uint32"}]`,
-			`[ { "type": "bytes32" } ]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+		want: uint32(1),
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
-			"bytes",
+	{
-			"",
+		def:  `[{"type": "uint32"}]`,
-		},
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-		{
+		want: uint16(0),
-			`[ { "type": "bytes32" } ]`,
+		err:  "abi: cannot unmarshal uint32 in to uint16",
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	{
-			"hash",
+		def:  `[{"type": "uint17"}]`,
-			"",
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-		},
+		want: uint16(0),
-		{
+		err:  "abi: cannot unmarshal *big.Int in to uint16",
-			`[ { "type": "bytes32" } ]`,
+	},
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	{
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+		def:  `[{"type": "uint17"}]`,
-			"interface",
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-			"",
+		want: big.NewInt(1),
-		},
+	},
-		{
+	{
-			`[ { "type": "function" } ]`,
+		def:  `[{"type": "int32"}]`,
-			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-			[24]byte{1},
+		want: int32(1),
-			"function",
+	},
-			"",
+	{
-		},
+		def:  `[{"type": "int32"}]`,
-	} {
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
-		abiDefinition := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
+		want: int16(0),
-		abi, err := JSON(strings.NewReader(abiDefinition))
+		err:  "abi: cannot unmarshal int32 in to int16",
+	},
+	{
+		def:  `[{"type": "int17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: int16(0),
+		err:  "abi: cannot unmarshal *big.Int in to int16",
+	},
+	{
+		def:  `[{"type": "int17"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
+		want: big.NewInt(1),
+	},
+	{
+		def:  `[{"type": "address"}]`,
+		enc:  "0000000000000000000000000100000000000000000000000000000000000000",
+		want: common.Address{1},
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: [32]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+	},
+	{
+		def:  `[{"type": "bytes"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
+	{
+		def:  `[{"type": "bytes"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: [32]byte{},
+		err:  "abi: cannot unmarshal []uint8 in to [32]uint8",
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200100000000000000000000000000000000000000000000000000000000000000",
+		want: []byte(nil),
+		err:  "abi: cannot unmarshal [32]uint8 in to []uint8",
+	},
+	{
+		def:  `[{"type": "bytes32"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: common.HexToHash("0100000000000000000000000000000000000000000000000000000000000000"),
+	},
+	{
+		def:  `[{"type": "function"}]`,
+		enc:  "0100000000000000000000000000000000000000000000000000000000000000",
+		want: [24]byte{1},
+	},
+	// slices
+	{
+		def:  `[{"type": "uint8[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint8{1, 2},
+	},
+	{
+		def:  `[{"type": "uint8[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint8{1, 2},
+	},
+	// multi dimensional, if these pass, all types that don't require length prefix should pass
+	{
+		def:  `[{"type": "uint8[][]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000E0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [][]uint8{{1, 2}, {1, 2}},
+	},
+	{
+		def:  `[{"type": "uint8[2][2]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2][2]uint8{{1, 2}, {1, 2}},
+	},
+	{
+		def:  `[{"type": "uint8[][2]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001",
+		want: [2][]uint8{{1}, {1}},
+	},
+	{
+		def:  `[{"type": "uint8[2][]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [][2]uint8{{1, 2}},
+	},
+	{
+		def:  `[{"type": "uint16[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint16{1, 2},
+	},
+	{
+		def:  `[{"type": "uint16[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint16{1, 2},
+	},
+	{
+		def:  `[{"type": "uint32[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint32{1, 2},
+	},
+	{
+		def:  `[{"type": "uint32[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint32{1, 2},
+	},
+	{
+		def:  `[{"type": "uint64[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []uint64{1, 2},
+	},
+	{
+		def:  `[{"type": "uint64[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]uint64{1, 2},
+	},
+	{
+		def:  `[{"type": "uint256[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []*big.Int{big.NewInt(1), big.NewInt(2)},
+	},
+	{
+		def:  `[{"type": "uint256[3]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		want: [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
+	},
+	{
+		def:  `[{"type": "int8[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int8{1, 2},
+	},
+	{
+		def:  `[{"type": "int8[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int8{1, 2},
+	},
+	{
+		def:  `[{"type": "int16[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int16{1, 2},
+	},
+	{
+		def:  `[{"type": "int16[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int16{1, 2},
+	},
+	{
+		def:  `[{"type": "int32[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int32{1, 2},
+	},
+	{
+		def:  `[{"type": "int32[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int32{1, 2},
+	},
+	{
+		def:  `[{"type": "int64[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []int64{1, 2},
+	},
+	{
+		def:  `[{"type": "int64[2]"}]`,
+		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: [2]int64{1, 2},
+	},
+	{
+		def:  `[{"type": "int256[]"}]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
+		want: []*big.Int{big.NewInt(1), big.NewInt(2)},
+	},
+	{
+		def:  `[{"type": "int256[3]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003",
+		want: [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)},
+	},
+}
+
+func TestUnpack(t *testing.T) {
+	for i, test := range unpackTests {
+		def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
+		abi, err := JSON(strings.NewReader(def))
 		if err != nil {
-			t.Errorf("%d failed. %v", i, err)
+			t.Fatalf("invalid ABI definition %s: %v", def, err)
+		}
+		encb, err := hex.DecodeString(test.enc)
+		if err != nil {
+			t.Fatalf("invalid hex: %s" + test.enc)
+		}
+		outptr := reflect.New(reflect.TypeOf(test.want))
+		err = abi.Unpack(outptr.Interface(), "method", encb)
+		if err := test.checkError(err); err != nil {
+			t.Errorf("test %d (%v) failed: %v", i, test.def, err)
 			continue
 		}
-
+		out := outptr.Elem().Interface()
-		var outvar interface{}
+		if !reflect.DeepEqual(test.want, out) {
-		switch test.outVar {
+			t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out)
-		case "bool":
-			var v bool
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint8":
-			var v uint8
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint16":
-			var v uint16
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint32":
-			var v uint32
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "uint64":
-			var v uint64
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int8":
-			var v int8
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int16":
-			var v int16
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int32":
-			var v int32
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "int64":
-			var v int64
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "*big.Int":
-			var v *big.Int
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "address":
-			var v common.Address
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "bytes":
-			var v []byte
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "hash":
-			var v common.Hash
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v.Bytes()[:]
-		case "function":
-			var v [24]byte
-			err = abi.Unpack(&v, "method", test.marshalledOutput)
-			outvar = v
-		case "interface":
-			err = abi.Unpack(&outvar, "method", test.marshalledOutput)
-		default:
-			t.Errorf("unsupported type '%v' please add it to the switch statement in this test", test.outVar)
-			continue
 		}
-
-		if err != nil && len(test.err) == 0 {
-			t.Errorf("%d failed. Expected no err but got: %v", i, err)
-			continue
-		}
-		if err == nil && len(test.err) != 0 {
-			t.Errorf("%d failed. Expected err: %v but got none", i, test.err)
-			continue
-		}
-		if err != nil && len(test.err) != 0 && err.Error() != test.err {
-			t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err)
-			continue
-		}
-
-		if err == nil {
-			if !reflect.DeepEqual(test.expectedOut, outvar) {
-				t.Errorf("%d failed. Output error: expected %v, got %v", i, test.expectedOut, outvar)
-			}
-		}
-	}
-}
-
-func TestUnpackSetInterfaceSlice(t *testing.T) {
-	var (
-		var1 = new(uint8)
-		var2 = new(uint8)
-	)
-	out := []interface{}{var1, var2}
-	abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint8"}, {"type":"uint8"}]}]`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...)
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if *var1 != 1 {
-		t.Error("expected var1 to be 1, got", *var1)
-	}
-	if *var2 != 2 {
-		t.Error("expected var2 to be 2, got", *var2)
-	}
-
-	out = []interface{}{var1}
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-
-	expErr := "abi: cannot marshal in to slices of unequal size (require: 2, got: 1)"
-	if err == nil || err.Error() != expErr {
-		t.Error("expected err:", expErr, "Got:", err)
-	}
-}
-
-func TestUnpackSetInterfaceArrayOutput(t *testing.T) {
-	var (
-		var1 = new([1]uint32)
-		var2 = new([1]uint32)
-	)
-	out := []interface{}{var1, var2}
-	abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint32[1]"}, {"type":"uint32[1]"}]}]`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...)
-	err = abi.Unpack(&out, "ints", marshalledReturn)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if *var1 != [1]uint32{1} {
-		t.Error("expected var1 to be [1], got", *var1)
-	}
-	if *var2 != [1]uint32{2} {
-		t.Error("expected var2 to be [2], got", *var2)
 	}
 }
 
@@ -337,101 +336,6 @@ func TestMultiReturnWithStruct(t *testing.T) {
 	}
 }
 
-func TestMultiReturnWithSlice(t *testing.T) {
-	const definition = `[
-	{ "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
-
-	abi, err := JSON(strings.NewReader(definition))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// using buff to make the code readable
-	buff := new(bytes.Buffer)
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005"))
-	stringOut := "hello"
-	buff.Write(common.RightPadBytes([]byte(stringOut), 32))
-
-	var inter []interface{}
-	err = abi.Unpack(&inter, "multi", buff.Bytes())
-	if err != nil {
-		t.Error(err)
-	}
-
-	if len(inter) != 2 {
-		t.Fatal("expected 2 results got", len(inter))
-	}
-
-	if num, ok := inter[0].(*big.Int); !ok || num.Cmp(big.NewInt(1)) != 0 {
-		t.Error("expected index 0 to be 1 got", num)
-	}
-
-	if str, ok := inter[1].(string); !ok || str != stringOut {
-		t.Error("expected index 1 to be", stringOut, "got", str)
-	}
-}
-
-func TestMarshalArrays(t *testing.T) {
-	const definition = `[
-	{ "name" : "bytes32", "constant" : false, "outputs": [ { "type": "bytes32" } ] },
-	{ "name" : "bytes10", "constant" : false, "outputs": [ { "type": "bytes10" } ] }
-	]`
-
-	abi, err := JSON(strings.NewReader(definition))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	output := common.LeftPadBytes([]byte{1}, 32)
-
-	var bytes10 [10]byte
-	err = abi.Unpack(&bytes10, "bytes32", output)
-	if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" {
-		t.Error("expected error or bytes32 not be assignable to bytes10:", err)
-	}
-
-	var bytes32 [32]byte
-	err = abi.Unpack(&bytes32, "bytes32", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(bytes32[:], output) {
-		t.Error("expected bytes32[31] to be 1 got", bytes32[31])
-	}
-
-	type (
-		B10 [10]byte
-		B32 [32]byte
-	)
-
-	var b10 B10
-	err = abi.Unpack(&b10, "bytes32", output)
-	if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" {
-		t.Error("expected error or bytes32 not be assignable to bytes10:", err)
-	}
-
-	var b32 B32
-	err = abi.Unpack(&b32, "bytes32", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(b32[:], output) {
-		t.Error("expected bytes32[31] to be 1 got", bytes32[31])
-	}
-
-	output[10] = 1
-	var shortAssignLong [32]byte
-	err = abi.Unpack(&shortAssignLong, "bytes10", output)
-	if err != nil {
-		t.Error("didn't expect error:", err)
-	}
-	if !bytes.Equal(output, shortAssignLong[:]) {
-		t.Errorf("expected %x to be %x", shortAssignLong, output)
-	}
-}
-
 func TestUnmarshal(t *testing.T) {
 	const definition = `[
 	{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },
@@ -450,6 +354,29 @@ func TestUnmarshal(t *testing.T) {
 	}
 	buff := new(bytes.Buffer)
 
+	// marshall mixed bytes (mixedBytes)
+	p0, p0Exp := []byte{}, common.Hex2Bytes("01020000000000000000")
+	p1, p1Exp := [32]byte{}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff")
+	mixedBytes := []interface{}{&p0, &p1}
+
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff"))
+	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000a"))
+	buff.Write(common.Hex2Bytes("0102000000000000000000000000000000000000000000000000000000000000"))
+
+	err = abi.Unpack(&mixedBytes, "mixedBytes", buff.Bytes())
+	if err != nil {
+		t.Error(err)
+	} else {
+		if bytes.Compare(p0, p0Exp) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", p0Exp, p0)
+		}
+
+		if bytes.Compare(p1[:], p1Exp) != 0 {
+			t.Errorf("unexpected value unpacked: want %x, got %x", p1Exp, p1)
+		}
+	}
+
 	// marshal int
 	var Int *big.Int
 	err = abi.Unpack(&Int, "int", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
@@ -473,6 +400,7 @@ func TestUnmarshal(t *testing.T) {
 	}
 
 	// marshal dynamic bytes max length 32
+	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	bytesOut := common.RightPadBytes([]byte("hello"), 32)
@@ -504,11 +432,11 @@ func TestUnmarshal(t *testing.T) {
 		t.Errorf("expected %x got %x", bytesOut, Bytes)
 	}
 
-	// marshall dynamic bytes max length 63
+	// marshall dynamic bytes max length 64
 	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
 	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f"))
-	bytesOut = common.RightPadBytes([]byte("hello"), 63)
+	bytesOut = common.RightPadBytes([]byte("hello"), 64)
 	buff.Write(bytesOut)
 
 	err = abi.Unpack(&Bytes, "bytes", buff.Bytes())
@@ -516,8 +444,8 @@ func TestUnmarshal(t *testing.T) {
 		t.Error(err)
 	}
 
-	if !bytes.Equal(Bytes, bytesOut) {
+	if !bytes.Equal(Bytes, bytesOut[:len(bytesOut)-1]) {
-		t.Errorf("expected %x got %x", bytesOut, Bytes)
+		t.Errorf("expected %x got %x", bytesOut[:len(bytesOut)-1], Bytes)
 	}
 
 	// marshal dynamic bytes output empty
@@ -569,29 +497,6 @@ func TestUnmarshal(t *testing.T) {
 		t.Error("expected error")
 	}
 
-	// marshal mixed bytes
-	buff.Reset()
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040"))
-	fixed := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")
-	buff.Write(fixed)
-	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020"))
-	bytesOut = common.RightPadBytes([]byte("hello"), 32)
-	buff.Write(bytesOut)
-
-	var out []interface{}
-	err = abi.Unpack(&out, "mixedBytes", buff.Bytes())
-	if err != nil {
-		t.Fatal("didn't expect error:", err)
-	}
-
-	if !bytes.Equal(bytesOut, out[0].([]byte)) {
-		t.Errorf("expected %x, got %x", bytesOut, out[0])
-	}
-
-	if !bytes.Equal(fixed, out[1].([]byte)) {
-		t.Errorf("expected %x, got %x", fixed, out[1])
-	}
-
 	buff.Reset()
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"))

accounts/keystore/account_cache.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -31,6 +30,7 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/accounts"
 	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/log"
+	"gopkg.in/fatih/set.v0"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -71,6 +71,7 @@ type accountCache struct {
 	byAddr   map[common.Address][]accounts.Account
 	throttle *time.Timer
 	notify   chan struct{}
+	fileC    fileCache
 }
 
 func newAccountCache(keydir string) (*accountCache, chan struct{}) {
@@ -78,6 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
 		keydir: keydir,
 		byAddr: make(map[common.Address][]accounts.Account),
 		notify: make(chan struct{}, 1),
+		fileC:  fileCache{all: set.NewNonTS()},
 	}
 	ac.watcher = newWatcher(ac)
 	return ac, ac.notify
@@ -127,6 +129,23 @@ func (ac *accountCache) delete(removed accounts.Account) {
 	}
 }
 
+// deleteByFile removes an account referenced by the given path.
+func (ac *accountCache) deleteByFile(path string) {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Path >= path })
+
+	if i < len(ac.all) && ac.all[i].URL.Path == path {
+		removed := ac.all[i]
+		ac.all = append(ac.all[:i], ac.all[i+1:]...)
+		if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
+			delete(ac.byAddr, removed.Address)
+		} else {
+			ac.byAddr[removed.Address] = ba
+		}
+	}
+}
+
 func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
@@ -167,15 +186,16 @@ func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
 	default:
 		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
 		copy(err.Matches, matches)
+		sort.Sort(accountsByURL(err.Matches))
 		return accounts.Account{}, err
 	}
 }
 
 func (ac *accountCache) maybeReload() {
 	ac.mu.Lock()
-	defer ac.mu.Unlock()
 
 	if ac.watcher.running {
+		ac.mu.Unlock()
 		return // A watcher is running and will keep the cache up-to-date.
 	}
 	if ac.throttle == nil {
@@ -184,12 +204,15 @@ func (ac *accountCache) maybeReload() {
 		select {
 		case <-ac.throttle.C:
 		default:
+			ac.mu.Unlock()
 			return // The cache was reloaded recently.
 		}
 	}
+	// No watcher running, start it.
 	ac.watcher.start()
-	ac.reload()
 	ac.throttle.Reset(minReloadInterval)
+	ac.mu.Unlock()
+	ac.scanAccounts()
 }
 
 func (ac *accountCache) close() {
@@ -205,80 +228,71 @@ func (ac *accountCache) close() {
 	ac.mu.Unlock()
 }
 
-// reload caches addresses of existing accounts.
+// scanAccounts checks if any changes have occurred on the filesystem, and
-// Callers must hold ac.mu.
+// updates the account cache accordingly
-func (ac *accountCache) reload() {
+func (ac *accountCache) scanAccounts() error {
-	accounts, err := ac.scan()
+	// Scan the entire folder metadata for file changes
+	creates, deletes, updates, err := ac.fileC.scan(ac.keydir)
 	if err != nil {
 		log.Debug("Failed to reload keystore contents", "err", err)
+		return err
 	}
-	ac.all = accounts
+	if creates.Size() == 0 && deletes.Size() == 0 && updates.Size() == 0 {
-	sort.Sort(ac.all)
+		return nil
-	for k := range ac.byAddr {
-		delete(ac.byAddr, k)
 	}
-	for _, a := range accounts {
+	// Create a helper method to scan the contents of the key files
-		ac.byAddr[a.Address] = append(ac.byAddr[a.Address], a)
+	var (
+		buf = new(bufio.Reader)
+		key struct {
+			Address string `json:"address"`
+		}
+	)
+	readAccount := func(path string) *accounts.Account {
+		fd, err := os.Open(path)
+		if err != nil {
+			log.Trace("Failed to open keystore file", "path", path, "err", err)
+			return nil
+		}
+		defer fd.Close()
+		buf.Reset(fd)
+		// Parse the address.
+		key.Address = ""
+		err = json.NewDecoder(buf).Decode(&key)
+		addr := common.HexToAddress(key.Address)
+		switch {
+		case err != nil:
+			log.Debug("Failed to decode keystore key", "path", path, "err", err)
+		case (addr == common.Address{}):
+			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
+		default:
+			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+		}
+		return nil
 	}
+	// Process all the file diffs
+	start := time.Now()
+
+	for _, p := range creates.List() {
+		if a := readAccount(p.(string)); a != nil {
+			ac.add(*a)
+		}
+	}
+	for _, p := range deletes.List() {
+		ac.deleteByFile(p.(string))
+	}
+	for _, p := range updates.List() {
+		path := p.(string)
+		ac.deleteByFile(path)
+		if a := readAccount(path); a != nil {
+			ac.add(*a)
+		}
+	}
+	end := time.Now()
+
 	select {
 	case ac.notify <- struct{}{}:
 	default:
 	}
-	log.Debug("Reloaded keystore contents", "accounts", len(ac.all))
+	log.Trace("Handled keystore changes", "time", end.Sub(start))
-}
+	return nil
-
-func (ac *accountCache) scan() ([]accounts.Account, error) {
-	files, err := ioutil.ReadDir(ac.keydir)
-	if err != nil {
-		return nil, err
-	}
-
-	var (
-		buf     = new(bufio.Reader)
-		addrs   []accounts.Account
-		keyJSON struct {
-			Address string `json:"address"`
-		}
-	)
-	for _, fi := range files {
-		path := filepath.Join(ac.keydir, fi.Name())
-		if skipKeyFile(fi) {
-			log.Trace("Ignoring file on account scan", "path", path)
-			continue
-		}
-		logger := log.New("path", path)
-
-		fd, err := os.Open(path)
-		if err != nil {
-			logger.Trace("Failed to open keystore file", "err", err)
-			continue
-		}
-		buf.Reset(fd)
-		// Parse the address.
-		keyJSON.Address = ""
-		err = json.NewDecoder(buf).Decode(&keyJSON)
-		addr := common.HexToAddress(keyJSON.Address)
-		switch {
-		case err != nil:
-			logger.Debug("Failed to decode keystore key", "err", err)
-		case (addr == common.Address{}):
-			logger.Debug("Failed to decode keystore key", "err", "missing or zero address")
-		default:
-			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
-		}
-		fd.Close()
-	}
-	return addrs, err
-}
-
-func skipKeyFile(fi os.FileInfo) bool {
-	// Skip editor backups and UNIX-style hidden files.
-	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
-		return true
-	}
-	// Skip misc special files, directories (yes, symlinks too).
-	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
-		return true
-	}
-	return false
 }

accounts/keystore/account_cache_test.go

@@ -18,6 +18,7 @@ package keystore
 
 import (
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -58,7 +59,7 @@ func TestWatchNewFile(t *testing.T) {
 
 	// Ensure the watcher is started before adding any files.
 	ks.Accounts()
-	time.Sleep(200 * time.Millisecond)
+	time.Sleep(1000 * time.Millisecond)
 
 	// Move in the files.
 	wantAccounts := make([]accounts.Account, len(cachetestAccounts))
@@ -295,3 +296,111 @@ func TestCacheFind(t *testing.T) {
 		}
 	}
 }
+
+func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
+	var list []accounts.Account
+	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
+		list = ks.Accounts()
+		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				return fmt.Errorf("wasn't notified of new accounts")
+			}
+			return nil
+		}
+		time.Sleep(d)
+	}
+	return fmt.Errorf("\ngot  %v\nwant %v", list, wantAccounts)
+}
+
+// TestUpdatedKeyfileContents tests that updating the contents of a keystore file
+// is noticed by the watcher, and the account cache is updated accordingly
+func TestUpdatedKeyfileContents(t *testing.T) {
+	t.Parallel()
+
+	// Create a temporary kesytore to test with
+	rand.Seed(time.Now().UnixNano())
+	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int()))
+	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
+
+	list := ks.Accounts()
+	if len(list) > 0 {
+		t.Error("initial account list not empty:", list)
+	}
+	time.Sleep(100 * time.Millisecond)
+
+	// Create the directory and copy a key file into it.
+	os.MkdirAll(dir, 0700)
+	defer os.RemoveAll(dir)
+	file := filepath.Join(dir, "aaa")
+
+	// Place one of our testfiles in there
+	if err := cp.CopyFile(file, cachetestAccounts[0].URL.Path); err != nil {
+		t.Fatal(err)
+	}
+
+	// ks should see the account.
+	wantAccounts := []accounts.Account{cachetestAccounts[0]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after forceCopyFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents
+	if err := forceCopyFile(file, cachetestAccounts[1].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[1]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("First replacement failed")
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after forceCopyFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents again
+	if err := forceCopyFile(file, cachetestAccounts[2].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[2]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("Second replacement failed")
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after ioutil.WriteFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents with crap
+	if err := ioutil.WriteFile(file, []byte("foo"), 0644); err != nil {
+		t.Fatal(err)
+		return
+	}
+	if err := waitForAccounts([]accounts.Account{}, ks); err != nil {
+		t.Errorf("Emptying account file failed")
+		t.Error(err)
+		return
+	}
+}
+
+// forceCopyFile is like cp.CopyFile, but doesn't complain if the destination exists.
+func forceCopyFile(dst, src string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(dst, data, 0644)
+}

accounts/keystore/file_cache.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+	set "gopkg.in/fatih/set.v0"
+)
+
+// fileCache is a cache of files seen during scan of keystore.
+type fileCache struct {
+	all     *set.SetNonTS // Set of all files from the keystore folder
+	lastMod time.Time     // Last time instance when a file was modified
+	mu      sync.RWMutex
+}
+
+// scan performs a new scan on the given directory, compares against the already
+// cached filenames, and returns file sets: creates, deletes, updates.
+func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Interface, error) {
+	t0 := time.Now()
+
+	// List all the failes from the keystore folder
+	files, err := ioutil.ReadDir(keyDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	t1 := time.Now()
+
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	// Iterate all the files and gather their metadata
+	all := set.NewNonTS()
+	mods := set.NewNonTS()
+
+	var newLastMod time.Time
+	for _, fi := range files {
+		// Skip any non-key files from the folder
+		path := filepath.Join(keyDir, fi.Name())
+		if skipKeyFile(fi) {
+			log.Trace("Ignoring file on account scan", "path", path)
+			continue
+		}
+		// Gather the set of all and fresly modified files
+		all.Add(path)
+
+		modified := fi.ModTime()
+		if modified.After(fc.lastMod) {
+			mods.Add(path)
+		}
+		if modified.After(newLastMod) {
+			newLastMod = modified
+		}
+	}
+	t2 := time.Now()
+
+	// Update the tracked files and return the three sets
+	deletes := set.Difference(fc.all, all)   // Deletes = previous - current
+	creates := set.Difference(all, fc.all)   // Creates = current - previous
+	updates := set.Difference(mods, creates) // Updates = modified - creates
+
+	fc.all, fc.lastMod = all, newLastMod
+	t3 := time.Now()
+
+	// Report on the scanning stats and return
+	log.Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2))
+	return creates, deletes, updates, nil
+}
+
+// skipKeyFile ignores editor backups, hidden files and folders/symlinks.
+func skipKeyFile(fi os.FileInfo) bool {
+	// Skip editor backups and UNIX-style hidden files.
+	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
+		return true
+	}
+	// Skip misc special files, directories (yes, symlinks too).
+	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
+		return true
+	}
+	return false
+}

accounts/keystore/keystore_passphrase.go

@@ -28,6 +28,7 @@ package keystore
 import (
 	"bytes"
 	"crypto/aes"
+	crand "crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -90,6 +91,12 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 	return key, nil
 }
 
+// StoreKey generates a key, encrypts with 'auth' and stores in the given directory
+func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, crand.Reader, auth)
+	return a.Address, err
+}
+
 func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) error {
 	keyjson, err := EncryptKey(key, auth, ks.scryptN, ks.scryptP)
 	if err != nil {

accounts/keystore/keystore_test.go

@@ -272,82 +272,104 @@ func TestWalletNotifierLifecycle(t *testing.T) {
 	t.Errorf("wallet notifier didn't terminate after unsubscribe")
 }
 
+type walletEvent struct {
+	accounts.WalletEvent
+	a accounts.Account
+}
+
 // Tests that wallet notifications and correctly fired when accounts are added
 // or deleted from the keystore.
 func TestWalletNotifications(t *testing.T) {
-	// Create a temporary kesytore to test with
 	dir, ks := tmpKeyStore(t, false)
 	defer os.RemoveAll(dir)
 
-	// Subscribe to the wallet feed
+	// Subscribe to the wallet feed and collect events.
-	updates := make(chan accounts.WalletEvent, 1)
+	var (
-	sub := ks.Subscribe(updates)
+		events  []walletEvent
+		updates = make(chan accounts.WalletEvent)
+		sub     = ks.Subscribe(updates)
+	)
 	defer sub.Unsubscribe()
+	go func() {
+		for {
+			select {
+			case ev := <-updates:
+				events = append(events, walletEvent{ev, ev.Wallet.Accounts()[0]})
+			case <-sub.Err():
+				close(updates)
+				return
+			}
+		}
+	}()
 
-	// Randomly add and remove account and make sure events and wallets are in sync
+	// Randomly add and remove accounts.
-	live := make(map[common.Address]accounts.Account)
+	var (
+		live       = make(map[common.Address]accounts.Account)
+		wantEvents []walletEvent
+	)
 	for i := 0; i < 1024; i++ {
-		// Execute a creation or deletion and ensure event arrival
 		if create := len(live) == 0 || rand.Int()%4 > 0; create {
 			// Add a new account and ensure wallet notifications arrives
 			account, err := ks.NewAccount("")
 			if err != nil {
 				t.Fatalf("failed to create test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if event.Kind != accounts.WalletArrived {
-					t.Errorf("non-arrival event on account creation")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on created wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet arrival event not fired on account creation")
-			}
 			live[account.Address] = account
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletArrived}, account})
 		} else {
-			// Select a random account to delete (crude, but works)
+			// Delete a random account.
 			var account accounts.Account
 			for _, a := range live {
 				account = a
 				break
 			}
-			// Remove an account and ensure wallet notifiaction arrives
 			if err := ks.Delete(account, ""); err != nil {
 				t.Fatalf("failed to delete test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if event.Kind != accounts.WalletDropped {
-					t.Errorf("non-drop event on account deletion")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on deleted wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet departure event not fired on account creation")
-			}
 			delete(live, account.Address)
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletDropped}, account})
 		}
-		// Retrieve the list of wallets and ensure it matches with our required live set
+	}
-		liveList := make([]accounts.Account, 0, len(live))
-		for _, account := range live {
-			liveList = append(liveList, account)
-		}
-		sort.Sort(accountsByURL(liveList))
 
-		wallets := ks.Wallets()
+	// Shut down the event collector and check events.
-		if len(liveList) != len(wallets) {
+	sub.Unsubscribe()
-			t.Errorf("wallet list doesn't match required accounts: have %v, want %v", wallets, liveList)
+	<-updates
-		} else {
+	checkAccounts(t, live, ks.Wallets())
-			for j, wallet := range wallets {
+	checkEvents(t, wantEvents, events)
-				if accs := wallet.Accounts(); len(accs) != 1 {
+}
-					t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+
-				} else if accs[0] != liveList[j] {
+// checkAccounts checks that all known live accounts are present in the wallet list.
-					t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+func checkAccounts(t *testing.T, live map[common.Address]accounts.Account, wallets []accounts.Wallet) {
-				}
+	if len(live) != len(wallets) {
+		t.Errorf("wallet list doesn't match required accounts: have %d, want %d", len(wallets), len(live))
+		return
+	}
+	liveList := make([]accounts.Account, 0, len(live))
+	for _, account := range live {
+		liveList = append(liveList, account)
+	}
+	sort.Sort(accountsByURL(liveList))
+	for j, wallet := range wallets {
+		if accs := wallet.Accounts(); len(accs) != 1 {
+			t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+		} else if accs[0] != liveList[j] {
+			t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+		}
+	}
+}
+
+// checkEvents checks that all events in 'want' are present in 'have'. Events may be present multiple times.
+func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
+	for _, wantEv := range want {
+		nmatch := 0
+		for ; len(have) > 0; nmatch++ {
+			if have[0].Kind != wantEv.Kind || have[0].a != wantEv.a {
+				break
 			}
+			have = have[1:]
+		}
+		if nmatch == 0 {
+			t.Fatalf("can't find event with Kind=%v for %x", wantEv.Kind, wantEv.a.Address)
 		}
 	}
 }

accounts/keystore/watch.go

@@ -70,7 +70,6 @@ func (w *watcher) loop() {
 		return
 	}
 	defer notify.Stop(w.ev)
-
 	logger.Trace("Started watching keystore folder")
 	defer logger.Trace("Stopped watching keystore folder")
 
@@ -82,32 +81,28 @@ func (w *watcher) loop() {
 	// When an event occurs, the reload call is delayed a bit so that
 	// multiple events arriving quickly only cause a single reload.
 	var (
-		debounce          = time.NewTimer(0)
+		debounceDuration = 500 * time.Millisecond
-		debounceDuration  = 500 * time.Millisecond
+		rescanTriggered  = false
-		inCycle, hadEvent bool
+		debounce         = time.NewTimer(0)
 	)
+	// Ignore initial trigger
+	if !debounce.Stop() {
+		<-debounce.C
+	}
 	defer debounce.Stop()
 	for {
 		select {
 		case <-w.quit:
 			return
 		case <-w.ev:
-			if !inCycle {
+			// Trigger the scan (with delay), if not already triggered
+			if !rescanTriggered {
 				debounce.Reset(debounceDuration)
-				inCycle = true
+				rescanTriggered = true
-			} else {
-				hadEvent = true
 			}
 		case <-debounce.C:
-			w.ac.mu.Lock()
+			w.ac.scanAccounts()
-			w.ac.reload()
+			rescanTriggered = false
-			w.ac.mu.Unlock()
-			if hadEvent {
-				debounce.Reset(debounceDuration)
-				inCycle, hadEvent = true, false
-			} else {
-				inCycle, hadEvent = false, false
-			}
 		}
 	}
 }

accounts/manager.go

@@ -41,6 +41,11 @@ type Manager struct {
 // NewManager creates a generic account manager to sign transaction via various
 // supported backends.
 func NewManager(backends ...Backend) *Manager {
+	// Retrieve the initial list of wallets from the backends and sort by URL
+	var wallets []Wallet
+	for _, backend := range backends {
+		wallets = merge(wallets, backend.Wallets()...)
+	}
 	// Subscribe to wallet notifications from all backends
 	updates := make(chan WalletEvent, 4*len(backends))
 
@@ -48,11 +53,6 @@ func NewManager(backends ...Backend) *Manager {
 	for i, backend := range backends {
 		subs[i] = backend.Subscribe(updates)
 	}
-	// Retrieve the initial list of wallets from the backends and sort by URL
-	var wallets []Wallet
-	for _, backend := range backends {
-		wallets = merge(wallets, backend.Wallets()...)
-	}
 	// Assemble the account manager and return
 	am := &Manager{
 		backends: make(map[reflect.Type][]Backend),

build/ci.go

@@ -19,12 +19,13 @@
 /*
 The ci command is called from Continuous Integration scripts.
 
-Usage: go run ci.go <command> <command flags/arguments>
+Usage: go run build/ci.go <command> <command flags/arguments>
 
 Available commands are:
 
    install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -misspell ] [ packages... ]                                      -- runs the tests
+   test       [ -coverage ] [ packages... ]                                                    -- runs the tests
+   lint                                                                                        -- runs certain pre-selected linters
    archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
@@ -120,7 +121,7 @@ var (
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "zesty"}
+	debDistros = []string{"trusty", "xenial", "zesty", "artful"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -146,6 +147,8 @@ func main() {
 		doInstall(os.Args[2:])
 	case "test":
 		doTest(os.Args[2:])
+	case "lint":
+		doLint(os.Args[2:])
 	case "archive":
 		doArchive(os.Args[2:])
 	case "debsrc":
@@ -196,7 +199,7 @@ func doInstall(cmdline []string) {
 		build.MustRun(goinstall)
 		return
 	}
-	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any prvious builds
+	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
 	if *arch == "arm" {
 		os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
 		for _, path := range filepath.SplitList(build.GOPATH()) {
@@ -280,7 +283,6 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 
 func doTest(cmdline []string) {
 	var (
-		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
@@ -294,10 +296,7 @@ func doTest(cmdline []string) {
 
 	// Run analysis tools before the tests.
 	build.MustRun(goTool("vet", packages...))
-	if *misspell {
+
-		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
-		// spellcheck(packages)
-	}
 	// Run the actual tests.
 	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
@@ -306,35 +305,39 @@ func doTest(cmdline []string) {
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
+
 	gotest.Args = append(gotest.Args, packages...)
 	build.MustRun(gotest)
 }
 
-// spellcheck runs the client9/misspell spellchecker package on all Go, Cgo and
+// runs gometalinter on requested packages
-// test files in the requested packages.
+func doLint(cmdline []string) {
-func spellcheck(packages []string) {
+	flag.CommandLine.Parse(cmdline)
-	// Ensure the spellchecker is available
-	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
 
-	// Windows chokes on long argument lists, check packages individually
+	packages := []string{"./..."}
-	for _, pkg := range packages {
+	if len(flag.CommandLine.Args()) > 0 {
-		// The spell checker doesn't work on packages, gather all .go files for it
+		packages = flag.CommandLine.Args()
-		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
+	}
-		if err != nil {
+	// Get metalinter and install all supported linters
-			log.Fatalf("source file listing failed: %v\n%s", err, string(out))
+	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v1"))
-		}
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), "--install")
-		// Retrieve the folder and assemble the source list
-		lines := strings.Split(string(out), "\n")
-		root := lines[0]
 
-		sources := make([]string, 0, len(lines)-1)
+	// Run fast linters batched together
-		for _, line := range lines[1:] {
+	configs := []string{
-			if line = strings.TrimSpace(line); line != "" {
+		"--vendor",
-				sources = append(sources, filepath.Join(root, line))
+		"--disable-all",
-			}
+		"--enable=vet",
-		}
+		"--enable=gofmt",
-		// Run the spell checker for this particular package
+		"--enable=misspell",
-		build.MustRunCommand(filepath.Join(GOBIN, "misspell"), append([]string{"-error"}, sources...)...)
+		"--enable=goconst",
+		"--min-occurrences=6", // for goconst
+	}
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
+
+	// Run slow linters one by one
+	for _, linter := range []string{"unconvert"} {
+		configs = []string{"--vendor", "--deadline=10m", "--disable-all", "--enable=" + linter}
+		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
 	}
 }
 

cmd/evm/staterunner.go

@@ -94,7 +94,8 @@ func stateTestCmd(ctx *cli.Context) error {
 		for _, st := range test.Subtests() {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
-			if state, err := test.Run(st, cfg); err != nil {
+			state, err := test.Run(st, cfg)
+			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
@@ -102,6 +103,11 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
+			// print state root for evmlab tracing (already committed above, so no need to delete objects again
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
+
 			results = append(results, *result)
 
 			// Print any structured logs collected

cmd/faucet/faucet.go

@@ -21,8 +21,10 @@ package main
 
 import (
 	"bytes"
+	"compress/zlib"
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"html/template"
@@ -33,6 +35,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -80,7 +83,8 @@ var (
 	captchaToken  = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side")
 	captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side")
 
-	logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+	noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication")
+	logFlag    = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
 )
 
 var (
@@ -129,6 +133,7 @@ func main() {
 		"Amounts":   amounts,
 		"Periods":   periods,
 		"Recaptcha": *captchaToken,
+		"NoAuth":    *noauthFlag,
 	})
 	if err != nil {
 		log.Crit("Failed to render the faucet template", "err", err)
@@ -181,10 +186,10 @@ func main() {
 
 // request represents an accepted funding request.
 type request struct {
-	Username string             `json:"username"` // GitHub user for displaying an avatar
+	Avatar  string             `json:"avatar"`  // Avatar URL to make the UI nicer
-	Account  common.Address     `json:"account"`  // Ethereum address being funded
+	Account common.Address     `json:"account"` // Ethereum address being funded
-	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
+	Time    time.Time          `json:"time"`    // Timestamp when the request was accepted
-	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+	Tx      *types.Transaction `json:"tx"`      // Transaction funding the account
 }
 
 // faucet represents a crypto faucet backed by an Ethereum light client.
@@ -299,6 +304,8 @@ func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
 // apiHandler handles requests for Ether grants and transaction statuses.
 func (f *faucet) apiHandler(conn *websocket.Conn) {
 	// Start tracking the connection and drop at the end
+	defer conn.Close()
+
 	f.lock.Lock()
 	f.conns = append(f.conns, conn)
 	f.lock.Unlock()
@@ -313,25 +320,50 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		}
 		f.lock.Unlock()
 	}()
-	// Send a few initial stats to the client
+	// Gather the initial stats from the network to report
-	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+	var (
-	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+		head    *types.Header
+		balance *big.Int
+		nonce   uint64
+		err     error
+	)
+	for {
+		// Attempt to retrieve the stats, may error on no faucet connectivity
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		head, err = f.client.HeaderByNumber(ctx, nil)
+		if err == nil {
+			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
+			if err == nil {
+				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
+			}
+		}
+		cancel()
 
-	websocket.JSON.Send(conn, map[string]interface{}{
+		// If stats retrieval failed, wait a bit and retry
+		if err != nil {
+			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+				log.Warn("Failed to send faucet error to client", "err", err)
+				return
+			}
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		// Initial stats reported successfully, proceed with user interaction
+		break
+	}
+	// Send over the initial stats and the latest header
+	if err = send(conn, map[string]interface{}{
 		"funds":    balance.Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
-	})
+	}, 3*time.Second); err != nil {
-	// Send the initial block to the client
+		log.Warn("Failed to send initial stats to client", "err", err)
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		return
-	header, err := f.client.HeaderByNumber(ctx, nil)
+	}
-	cancel()
+	if err = send(conn, head, 3*time.Second); err != nil {
-
+		log.Warn("Failed to send initial header to client", "err", err)
-	if err != nil {
+		return
-		log.Error("Failed to retrieve latest header", "err", err)
-	} else {
-		websocket.JSON.Send(conn, header)
 	}
 	// Keep reading requests from the websocket until the connection breaks
 	for {
@@ -341,18 +373,25 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			Tier    uint   `json:"tier"`
 			Captcha string `json:"captcha"`
 		}
-		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+		if err = websocket.JSON.Receive(conn, &msg); err != nil {
 			return
 		}
-		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
+		if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://gist.github.com/") && !strings.HasPrefix(msg.URL, "https://twitter.com/") &&
-			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+			!strings.HasPrefix(msg.URL, "https://plus.google.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
+			if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil {
+				log.Warn("Failed to send URL error to client", "err", err)
+				return
+			}
 			continue
 		}
 		if msg.Tier >= uint(*tiersFlag) {
-			websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"})
+			if err = sendError(conn, errors.New("Invalid funding tier requested")); err != nil {
+				log.Warn("Failed to send tier error to client", "err", err)
+				return
+			}
 			continue
 		}
-		log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier)
+		log.Info("Faucet funds requested", "url", msg.URL, "tier", msg.Tier)
 
 		// If captcha verifications are enabled, make sure we're not dealing with a robot
 		if *captchaToken != "" {
@@ -362,7 +401,10 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 
 			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send captcha post error to client", "err", err)
+					return
+				}
 				continue
 			}
 			var result struct {
@@ -372,74 +414,61 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			err = json.NewDecoder(res.Body).Decode(&result)
 			res.Body.Close()
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send captcha decode error to client", "err", err)
+					return
+				}
 				continue
 			}
 			if !result.Success {
 				log.Warn("Captcha verification failed", "err", string(result.Errors))
-				websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"})
+				if err = sendError(conn, errors.New("Beep-bop, you're a robot!")); err != nil {
+					log.Warn("Failed to send captcha failure to client", "err", err)
+					return
+				}
 				continue
 			}
 		}
-		// Retrieve the gist from the GitHub Gist APIs
+		// Retrieve the Ethereum address to fund, the requesting user and a profile picture
-		parts := strings.Split(msg.URL, "/")
+		var (
-		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+			username string
-		if *githubUser != "" {
+			avatar   string
-			req.SetBasicAuth(*githubUser, *githubToken)
+			address  common.Address
-		}
+		)
-		res, err := http.DefaultClient.Do(req)
+		switch {
-		if err != nil {
+		case strings.HasPrefix(msg.URL, "https://gist.github.com/"):
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			if err = sendError(conn, errors.New("GitHub authentication discontinued at the official request of GitHub")); err != nil {
-			continue
+				log.Warn("Failed to send GitHub deprecation to client", "err", err)
-		}
+				return
-		var gist struct {
-			Owner struct {
-				Login string `json:"login"`
-			} `json:"owner"`
-			Files map[string]struct {
-				Content string `json:"content"`
-			} `json:"files"`
-		}
-		err = json.NewDecoder(res.Body).Decode(&gist)
-		res.Body.Close()
-		if err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
-			continue
-		}
-		if gist.Owner.Login == "" {
-			websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"})
-			continue
-		}
-		// Iterate over all the files and look for Ethereum addresses
-		var address common.Address
-		for _, file := range gist.Files {
-			content := strings.TrimSpace(file.Content)
-			if len(content) == 2+common.AddressLength*2 {
-				address = common.HexToAddress(content)
 			}
+			continue
+		case strings.HasPrefix(msg.URL, "https://twitter.com/"):
+			username, avatar, address, err = authTwitter(msg.URL)
+		case strings.HasPrefix(msg.URL, "https://plus.google.com/"):
+			username, avatar, address, err = authGooglePlus(msg.URL)
+		case strings.HasPrefix(msg.URL, "https://www.facebook.com/"):
+			username, avatar, address, err = authFacebook(msg.URL)
+		case *noauthFlag:
+			username, avatar, address, err = authNoAuth(msg.URL)
+		default:
+			err = errors.New("Something funky happened, please open an issue at https://github.com/EthereumCommonwealth/go-callisto/issues")
 		}
-		if address == (common.Address{}) {
+		if err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
+			if err = sendError(conn, err); err != nil {
+				log.Warn("Failed to send prefix error to client", "err", err)
+				return
+			}
 			continue
 		}
-		// Validate the user's existence since the API is unhelpful here
+		log.Info("Faucet request valid", "url", msg.URL, "tier", msg.Tier, "user", username, "address", address)
-		if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
-			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
-			continue
-		}
-		res.Body.Close()
 
-		if res.StatusCode != 200 {
-			websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"})
-			continue
-		}
 		// Ensure the user didn't request funds too recently
 		f.lock.Lock()
 		var (
 			fund    bool
 			timeout time.Time
 		)
-		if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) {
+		if timeout = f.timeouts[username]; time.Now().After(timeout) {
 			// User wasn't funded recently, create the funding transaction
 			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
 			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
@@ -448,33 +477,45 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil)
 			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
 			if err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
 				f.lock.Unlock()
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send transaction creation error to client", "err", err)
+					return
+				}
 				continue
 			}
 			// Submit the transaction and mark as funded if successful
 			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
-				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
 				f.lock.Unlock()
+				if err = sendError(conn, err); err != nil {
+					log.Warn("Failed to send transaction transmission error to client", "err", err)
+					return
+				}
 				continue
 			}
 			f.reqs = append(f.reqs, &request{
-				Username: gist.Owner.Login,
+				Avatar:  avatar,
-				Account:  address,
+				Account: address,
-				Time:     time.Now(),
+				Time:    time.Now(),
-				Tx:       signed,
+				Tx:      signed,
 			})
-			f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			f.timeouts[username] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
 			fund = true
 		}
 		f.lock.Unlock()
 
 		// Send an error if too frequent funding, othewise a success
 		if !fund {
-			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))})
+			if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))); err != nil {
+				log.Warn("Failed to send funding error to client", "err", err)
+				return
+			}
 			continue
 		}
-		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		if err = sendSuccess(conn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil {
+			log.Warn("Failed to send funding success to client", "err", err)
+			return
+		}
 		select {
 		case f.update <- struct{}{}:
 		default:
@@ -497,11 +538,31 @@ func (f *faucet) loop() {
 		select {
 		case head := <-heads:
 			// New chain head arrived, query the current stats and stream to clients
-			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+			var (
-			balance = new(big.Int).Div(balance, ether)
+				balance *big.Int
+				nonce   uint64
+				price   *big.Int
+				err     error
+			)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
+			if err == nil {
+				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
+				if err == nil {
+					price, err = f.client.SuggestGasPrice(ctx)
+				}
+			}
+			cancel()
 
-			price, _ := f.client.SuggestGasPrice(context.Background())
+			// If querying the data failed, try for the next block
-			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+			if err != nil {
+				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
+				continue
+			} else {
+				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
+			}
+			// Faucet state retrieved, update locally and send to clients
+			balance = new(big.Int).Div(balance, ether)
 
 			f.lock.Lock()
 			f.price, f.nonce = price, nonce
@@ -512,17 +573,17 @@ func (f *faucet) loop() {
 
 			f.lock.RLock()
 			for _, conn := range f.conns {
-				if err := websocket.JSON.Send(conn, map[string]interface{}{
+				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
 					"peers":    f.stack.Server().PeerCount(),
 					"requests": f.reqs,
-				}); err != nil {
+				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)
 					conn.Close()
 					continue
 				}
-				if err := websocket.JSON.Send(conn, head); err != nil {
+				if err := send(conn, head, time.Second); err != nil {
 					log.Warn("Failed to send header to client", "err", err)
 					conn.Close()
 				}
@@ -533,7 +594,7 @@ func (f *faucet) loop() {
 			// Pending requests updated, stream to clients
 			f.lock.RLock()
 			for _, conn := range f.conns {
-				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+				if err := send(conn, map[string]interface{}{"requests": f.reqs}, time.Second); err != nil {
 					log.Warn("Failed to send requests to client", "err", err)
 					conn.Close()
 				}
@@ -542,3 +603,195 @@ func (f *faucet) loop() {
 		}
 	}
 }
+
+// sends transmits a data packet to the remote end of the websocket, but also
+// setting a write deadline to prevent waiting forever on the node.
+func send(conn *websocket.Conn, value interface{}, timeout time.Duration) error {
+	if timeout == 0 {
+		timeout = 60 * time.Second
+	}
+	conn.SetWriteDeadline(time.Now().Add(timeout))
+	return websocket.JSON.Send(conn, value)
+}
+
+// sendError transmits an error to the remote end of the websocket, also setting
+// the write deadline to 1 second to prevent waiting forever.
+func sendError(conn *websocket.Conn, err error) error {
+	return send(conn, map[string]string{"error": err.Error()}, time.Second)
+}
+
+// sendSuccess transmits a success message to the remote end of the websocket, also
+// setting the write deadline to 1 second to prevent waiting forever.
+func sendSuccess(conn *websocket.Conn, msg string) error {
+	return send(conn, map[string]string{"success": msg}, time.Second)
+}
+
+// authGitHub tries to authenticate a faucet request using GitHub gists, returning
+// the username, avatar URL and Ethereum address to fund on success.
+func authGitHub(url string) (string, string, common.Address, error) {
+	// Retrieve the gist from the GitHub Gist APIs
+	parts := strings.Split(url, "/")
+	req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+	if *githubUser != "" {
+		req.SetBasicAuth(*githubUser, *githubToken)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	var gist struct {
+		Owner struct {
+			Login string `json:"login"`
+		} `json:"owner"`
+		Files map[string]struct {
+			Content string `json:"content"`
+		} `json:"files"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&gist)
+	res.Body.Close()
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	if gist.Owner.Login == "" {
+		return "", "", common.Address{}, errors.New("Anonymous Gists not allowed")
+	}
+	// Iterate over all the files and look for Ethereum addresses
+	var address common.Address
+	for _, file := range gist.Files {
+		content := strings.TrimSpace(file.Content)
+		if len(content) == 2+common.AddressLength*2 {
+			address = common.HexToAddress(content)
+		}
+	}
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	// Validate the user's existence since the API is unhelpful here
+	if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
+		return "", "", common.Address{}, err
+	}
+	res.Body.Close()
+
+	if res.StatusCode != 200 {
+		return "", "", common.Address{}, errors.New("Invalid user... boom!")
+	}
+	// Everything passed validation, return the gathered infos
+	return gist.Owner.Login + "@github", fmt.Sprintf("https://github.com/%s.png?size=64", gist.Owner.Login), address, nil
+}
+
+// authTwitter tries to authenticate a faucet request using Twitter posts, returning
+// the username, avatar URL and Ethereum address to fund on success.
+func authTwitter(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "status" {
+		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Twitter's API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	reader, err := zlib.NewReader(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@twitter", avatar, address, nil
+}
+
+// authGooglePlus tries to authenticate a faucet request using GooglePlus posts,
+// returning the username, avatar URL and Ethereum address to fund on success.
+func authGooglePlus(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "posts" {
+		return "", "", common.Address{}, errors.New("Invalid Google+ post URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Google's API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+googleusercontent.com[^\"]+photo.jpg)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@google+", avatar, address, nil
+}
+
+// authFacebook tries to authenticate a faucet request using Facebook posts,
+// returning the username, avatar URL and Ethereum address to fund on success.
+func authFacebook(url string) (string, string, common.Address, error) {
+	// Ensure the user specified a meaningful URL, no fancy nonsense
+	parts := strings.Split(url, "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "posts" {
+		return "", "", common.Address{}, errors.New("Invalid Facebook post URL")
+	}
+	username := parts[len(parts)-3]
+
+	// Facebook's Graph API isn't really friendly with direct links. Still, we don't
+	// want to do ask read permissions from users, so just load the public posts and
+	// scrape it for the Ethereum address and profile URL.
+	res, err := http.Get(url)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	var avatar string
+	if parts = regexp.MustCompile("src=\"([^\"]+fbcdn.net[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
+		avatar = parts[1]
+	}
+	return username + "@facebook", avatar, address, nil
+}
+
+// authNoAuth tries to interpret a faucet request as a plain Ethereum address,
+// without actually performing any remote authentication. This mode is prone to
+// Byzantine attack, so only ever use for truly private networks.
+func authNoAuth(url string) (string, string, common.Address, error) {
+	address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(url))
+	if address == (common.Address{}) {
+		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	return address.Hex() + "@noauth", "", address, nil
+}

cmd/faucet/faucet.html

@@ -5,7 +5,7 @@
 		<meta http-equiv="X-UA-Compatible" content="IE=edge">
 		<meta name="viewport" content="width=device-width, initial-scale=1">
 
-		<title>{{.Network}}: GitHub Faucet</title>
+		<title>{{.Network}}: Authenticated Faucet</title>
 
 		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
 		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
@@ -43,13 +43,13 @@
 			<div class="container">
 				<div class="row" style="margin-bottom: 16px;">
 					<div class="col-lg-12">
-						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} Authenticated Faucet</h1>
 					</div>
 				</div>
 				<div class="row">
 					<div class="col-lg-8 col-lg-offset-2">
 						<div class="input-group">
-							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<input id="url" name="url" type="text" class="form-control" placeholder="Social network URL containing your Ethereum address...">
 							<span class="input-group-btn">
 								<button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Give me Ether	<i class="fa fa-caret-down" aria-hidden="true"></i></button>
 				        <ul class="dropdown-menu dropdown-menu-right">{{range $idx, $amount := .Amounts}}
@@ -80,8 +80,23 @@
 				<div class="row" style="margin-top: 32px;">
 					<div class="col-lg-12">
 						<h3>How does this work?</h3>
-						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.</p>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to common 3rd party social network accounts. Anyone having a Twitter, Google+ or Facebook account may request funds within the permitted limits.</p>
-						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+						<dl class="dl-horizontal">
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-twitter" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Twitter, make a <a href="https://twitter.com/intent/tweet?text=Requesting%20faucet%20funds%20into%200x0000000000000000000000000000000000000000%20on%20the%20%23{{.Network}}%20%23Ethereum%20test%20network." target="_about:blank">tweet</a> with your Ethereum address pasted into the contents (surrounding text doesn't matter).<br/>Copy-paste the <a href="https://support.twitter.com/articles/80586" target="_about:blank">tweets URL</a> into the above input box and fire away!</dd>
+
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-google-plus-official" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Google Plus, publish a new <strong>public</strong> post with your Ethereum address embedded into the content (surrounding text doesn't matter).<br/>Copy-paste the posts URL into the above input box and fire away!</dd>
+
+							<dt style="width: auto; margin-left: 40px;"><i class="fa fa-facebook" aria-hidden="true" style="font-size: 36px;"></i></dt>
+							<dd style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds via Facebook, publish a new <strong>public</strong> post with your Ethereum address embedded into the content (surrounding text doesn't matter).<br/>Copy-paste the <a href="https://www.facebook.com/help/community/question/?id=282662498552845" target="_about:blank">posts URL</a> into the above input box and fire away!</dd>
+
+							{{if .NoAuth}}
+								<dt class="text-danger" style="width: auto; margin-left: 40px;"><i class="fa fa-unlock-alt" aria-hidden="true" style="font-size: 36px;"></i></dt>
+								<dd class="text-danger" style="margin-left: 88px; margin-bottom: 10px;"></i> To request funds <strong>without authentication</strong>, simply copy-paste your Ethereum address into the above input box (surrounding text doesn't matter) and fire away.<br/>This mode is susceptible to Byzantine attacks. Only use for debugging or private networks!</dd>
+							{{end}}
+						</dl>
+						<p>You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
 						{{if .Recaptcha}}<em>The faucet is running invisible reCaptcha protection against bots.</em>{{end}}
 					</div>
 				</div>
@@ -93,20 +108,27 @@
 			var attempt = 0;
 			var server;
 			var tier = 0;
+			var requests = [];
 
+			// Define a function that creates closures to drop old requests
+			var dropper = function(hash) {
+				return function() {
+					for (var i=0; i<requests.length; i++) {
+						if (requests[i].tx.hash == hash) {
+							requests.splice(i, 1);
+							break;
+						}
+					}
+				}
+			};
 			// Define the function that submits a gist url to the server
 			var submit = function({{if .Recaptcha}}captcha{{end}}) {
-				server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
+				server.send(JSON.stringify({url: $("#url")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
 				grecaptcha.reset();{{end}}
 			};
 			// Define a method to reconnect upon server loss
 			var reconnect = function() {
-				if (attempt % 2 == 0) {
+				server = new WebSocket(((window.location.protocol === "https:") ? "wss://" : "ws://") + window.location.host + "/api");
-					server = new WebSocket("wss://" + location.host + "/api");
-				} else {
-					server = new WebSocket("ws://" + location.host + "/api");
-				}
-				attempt++;
 
 				server.onmessage = function(event) {
 					var msg = JSON.parse(event.data);
@@ -127,21 +149,85 @@
 						$("#block").text(parseInt(msg.number, 16));
 					}
 					if (msg.error !== undefined) {
-						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+						noty({layout: 'topCenter', text: msg.error, type: 'error', timeout: 5000, progressBar: true});
 					}
 					if (msg.success !== undefined) {
-						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+						noty({layout: 'topCenter', text: msg.success, type: 'success', timeout: 5000, progressBar: true});
 					}
 					if (msg.requests !== undefined && msg.requests !== null) {
+						// Mark all previous requests missing as done
+						for (var i=0; i<requests.length; i++) {
+							if (msg.requests.length > 0 && msg.requests[0].tx.hash == requests[i].tx.hash) {
+								break;
+							}
+							if (requests[i].time != "") {
+								requests[i].time = "";
+								setTimeout(dropper(requests[i].tx.hash), 3000);
+							}
+						}
+						// Append any new requests into our local collection
+						var common = -1;
+						if (requests.length > 0) {
+							for (var i=0; i<msg.requests.length; i++) {
+								if (requests[requests.length-1].tx.hash == msg.requests[i].tx.hash) {
+									common = i;
+									break;
+								}
+							}
+						}
+						for (var i=common+1; i<msg.requests.length; i++) {
+							requests.push(msg.requests[i]);
+						}
+						// Iterate over our entire local collection and re-render the funding table
 						var content = "";
-						for (var i=0; i<msg.requests.length; i++) {
+						for (var i=0; i<requests.length; i++) {
-							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+							var done    = requests[i].time == "";
+							var elapsed = moment().unix()-moment(requests[i].time).unix();
+
+							content += "<tr id='" + requests[i].tx.hash + "'>";
+							content += "  <td><div style=\"background: url('" + requests[i].avatar + "'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td>";
+							content += "  <td><pre>" + requests[i].account + "</pre></td>";
+							content += "  <td style=\"width: 100%; text-align: center; vertical-align: middle;\">";
+							if (done) {
+								content += "    funded";
+							} else {
+								content += "    <span id='time-" + i + "' class='timer'>" + moment.duration(-elapsed, 'seconds').humanize(true) + "</span>";
+							}
+							content += "    <div class='progress' style='height: 4px; margin: 0;'>";
+							if (done) {
+								content += "      <div class='progress-bar progress-bar-success' role='progressbar' aria-valuenow='30' style='width:100%;'></div>";
+							} else if (elapsed > 30) {
+								content += "      <div class='progress-bar progress-bar-danger progress-bar-striped active' role='progressbar' aria-valuenow='30' style='width:100%;'></div>";
+							} else {
+								content += "      <div class='progress-bar progress-bar-striped active' role='progressbar' aria-valuenow='" + elapsed + "' style='width:" + (elapsed * 100 / 30) + "%;'></div>";
+							}
+							content += "    </div>";
+							content += "  </td>";
+							content += "</tr>";
 						}
 						$("#requests").html("<tbody>" + content + "</tbody>");
 					}
 				}
 				server.onclose = function() { setTimeout(reconnect, 3000); };
 			}
+			// Start a UI updater to push the progress bars forward until they are done
+			setInterval(function() {
+				$('.progress-bar').each(function() {
+					var progress = Number($(this).attr('aria-valuenow')) + 1;
+					if (progress < 30) {
+						$(this).attr('aria-valuenow', progress);
+						$(this).css('width', (progress * 100 / 30) + '%');
+					} else if (progress == 30) {
+						$(this).css('width', '100%');
+						$(this).addClass("progress-bar-danger");
+					}
+				})
+				$('.timer').each(function() {
+					var index = Number($(this).attr('id').substring(5));
+					$(this).html(moment.duration(moment(requests[index].time).unix()-moment().unix(), 'seconds').humanize(true));
+				})
+			}, 1000);
+
 			// Establish a websocket connection to the API server
 			reconnect();
 		</script>{{if .Recaptcha}}

cmd/geth/accountcmd.go

@@ -291,15 +291,28 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack, _ := makeConfigNode(ctx)
+	cfg := gethConfig{Node: defaultNodeConfig()}
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	scryptN, scryptP, keydir, err := cfg.Node.AccountConfig()
+
+	if err != nil {
+		utils.Fatalf("Failed to read configuration: %v", err)
+	}
+
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
-	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	address, err := keystore.StoreKey(keydir, password, scryptN, scryptP)
-	account, err := ks.NewAccount(password)
+
 	if err != nil {
 		utils.Fatalf("Failed to create account: %v", err)
 	}
-	fmt.Printf("Address: {%x}\n", account.Address)
+	fmt.Printf("Address: {%x}\n", address)
 	return nil
 }
 

cmd/geth/accountcmd_test.go

@@ -134,7 +134,7 @@ Fatal: could not decrypt key with given passphrase
 func TestUnlockFlag(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
 	geth.Expect(`
@@ -158,7 +158,7 @@ Passphrase: {{.InputLine "foobar"}}
 func TestUnlockFlagWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer geth.ExpectExit()
 	geth.Expect(`
@@ -177,7 +177,7 @@ Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could
 func TestUnlockFlagMultiIndex(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "0,2",
 		"js", "testdata/empty.js")
 	geth.Expect(`
@@ -204,7 +204,7 @@ Passphrase: {{.InputLine "foobar"}}
 func TestUnlockFlagPasswordFile(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--password", "testdata/passwords.txt", "--unlock", "0,2",
 		"js", "testdata/empty.js")
 	geth.ExpectExit()
@@ -224,7 +224,7 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
+		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
 	defer geth.ExpectExit()
 	geth.Expect(`
@@ -235,7 +235,7 @@ Fatal: Failed to unlock account 0 (could not decrypt key with given passphrase)
 func TestUnlockFlagAmbiguous(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
+		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
 	defer geth.ExpectExit()
@@ -273,7 +273,7 @@ In order to avoid this warning, you need to remove the following duplicate key f
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
+		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer geth.ExpectExit()
 

cmd/geth/chaincmd.go

@@ -31,7 +31,9 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/core"
 	"github.com/EthereumCommonwealth/go-callisto/core/state"
 	"github.com/EthereumCommonwealth/go-callisto/core/types"
+	"github.com/EthereumCommonwealth/go-callisto/eth/downloader"
 	"github.com/EthereumCommonwealth/go-callisto/ethdb"
+	"github.com/EthereumCommonwealth/go-callisto/event"
 	"github.com/EthereumCommonwealth/go-callisto/log"
 	"github.com/EthereumCommonwealth/go-callisto/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -90,6 +92,23 @@ Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
 if already existing.`,
+	}
+	copydbCommand = cli.Command{
+		Action:    utils.MigrateFlags(copyDb),
+		Name:      "copydb",
+		Usage:     "Create a local chain from a target chaindata folder",
+		ArgsUsage: "<sourceChaindataDir>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.SyncModeFlag,
+			utils.FakePoWFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+		Description: `
+The first argument must be the directory containing the blockchain to download from`,
 	}
 	removedbCommand = cli.Command{
 		Action:    utils.MigrateFlags(removeDB),
@@ -268,6 +287,54 @@ func exportChain(ctx *cli.Context) error {
 	return nil
 }
 
+func copyDb(ctx *cli.Context) error {
+	// Ensure we have a source chain directory to copy
+	if len(ctx.Args()) != 1 {
+		utils.Fatalf("Source chaindata directory path argument missing")
+	}
+	// Initialize a new chain for the running node to sync into
+	stack := makeFullNode(ctx)
+	chain, chainDb := utils.MakeChain(ctx, stack)
+
+	syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
+	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
+
+	// Create a source peer to satisfy downloader requests from
+	db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256)
+	if err != nil {
+		return err
+	}
+	hc, err := core.NewHeaderChain(db, chain.Config(), chain.Engine(), func() bool { return false })
+	if err != nil {
+		return err
+	}
+	peer := downloader.NewFakePeer("local", db, hc, dl)
+	if err = dl.RegisterPeer("local", 63, peer); err != nil {
+		return err
+	}
+	// Synchronise with the simulated peer
+	start := time.Now()
+
+	currentHeader := hc.CurrentHeader()
+	if err = dl.Synchronise("local", currentHeader.Hash(), hc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()), syncmode); err != nil {
+		return err
+	}
+	for dl.Synchronising() {
+		time.Sleep(10 * time.Millisecond)
+	}
+	fmt.Printf("Database copy done in %v\n", time.Since(start))
+
+	// Compact the entire database to remove any sync overhead
+	start = time.Now()
+	fmt.Println("Compacting entire database...")
+	if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil {
+		utils.Fatalf("Compaction failed: %v", err)
+	}
+	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
+
+	return nil
+}
+
 func removeDB(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 

cmd/geth/config.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/EthereumCommonwealth/go-callisto/cmd/utils"
 	"github.com/EthereumCommonwealth/go-callisto/contracts/release"
+	"github.com/EthereumCommonwealth/go-callisto/dashboard"
 	"github.com/EthereumCommonwealth/go-callisto/eth"
 	"github.com/EthereumCommonwealth/go-callisto/node"
 	"github.com/EthereumCommonwealth/go-callisto/params"
@@ -76,10 +77,11 @@ type ethstatsConfig struct {
 }
 
 type gethConfig struct {
-	Eth      eth.Config
+	Eth       eth.Config
-	Shh      whisper.Config
+	Shh       whisper.Config
-	Node     node.Config
+	Node      node.Config
-	Ethstats ethstatsConfig
+	Ethstats  ethstatsConfig
+	Dashboard dashboard.Config
 }
 
 func loadConfig(file string, cfg *gethConfig) error {
@@ -110,9 +112,10 @@ func defaultNodeConfig() node.Config {
 func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	// Load defaults.
 	cfg := gethConfig{
-		Eth:  eth.DefaultConfig,
+		Eth:       eth.DefaultConfig,
-		Shh:  whisper.DefaultConfig,
+		Shh:       whisper.DefaultConfig,
-		Node: defaultNodeConfig(),
+		Node:      defaultNodeConfig(),
+		Dashboard: dashboard.DefaultConfig,
 	}
 
 	// Load config file.
@@ -134,6 +137,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	}
 
 	utils.SetShhConfig(ctx, stack, &cfg.Shh)
+	utils.SetDashboardConfig(ctx, &cfg.Dashboard)
 
 	return stack, cfg
 }
@@ -153,9 +157,12 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 
 	utils.RegisterEthService(stack, &cfg.Eth)
 
+	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
+		utils.RegisterDashboardService(stack, &cfg.Dashboard)
+	}
 	// Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode
 	shhEnabled := enableWhisper(ctx)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DeveloperFlag.Name)
 	if shhEnabled || shhAutoEnabled {
 		if ctx.GlobalIsSet(utils.WhisperMaxMessageSizeFlag.Name) {
 			cfg.Shh.MaxMessageSize = uint32(ctx.Int(utils.WhisperMaxMessageSizeFlag.Name))

cmd/geth/consolecmd.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"os/signal"
 	"strings"
@@ -112,7 +113,11 @@ func localConsole(ctx *cli.Context) error {
 // console to it.
 func remoteConsole(ctx *cli.Context) error {
 	// Attach to a remotely running geth instance and start the JavaScript console
-	client, err := dialRPC(ctx.Args().First())
+	endpoint := ctx.Args().First()
+	if endpoint == "" && ctx.GlobalIsSet(utils.DataDirFlag.Name) {
+		endpoint = fmt.Sprintf("%s/geth.ipc", ctx.GlobalString(utils.DataDirFlag.Name))
+	}
+	client, err := dialRPC(endpoint)
 	if err != nil {
 		utils.Fatalf("Unable to attach to remote geth: %v", err)
 	}

cmd/geth/main.go

@@ -61,6 +61,11 @@ var (
 		utils.DataDirFlag,
 		utils.KeyStoreDirFlag,
 		utils.NoUSBFlag,
+		utils.DashboardEnabledFlag,
+		utils.DashboardAddrFlag,
+		utils.DashboardPortFlag,
+		utils.DashboardRefreshFlag,
+		utils.DashboardAssetsFlag,
 		utils.EthashCacheDirFlag,
 		utils.EthashCachesInMemoryFlag,
 		utils.EthashCachesOnDiskFlag,
@@ -99,7 +104,8 @@ var (
 		utils.NetrestrictFlag,
 		utils.NodeKeyFileFlag,
 		utils.NodeKeyHexFlag,
-		utils.DevModeFlag,
+		utils.DeveloperFlag,
+		utils.DeveloperPeriodFlag,
 		utils.TestnetFlag,
 		utils.RinkebyFlag,
 		utils.VMEnableDebugFlag,
@@ -146,6 +152,7 @@ func init() {
 		initCommand,
 		importCommand,
 		exportCommand,
+		copydbCommand,
 		removedbCommand,
 		dumpCommand,
 		// See monitorcmd.go:
@@ -269,7 +276,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		}
 	}()
 	// Start auxiliary services if enabled
-	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
+	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
 		// Mining only makes sense if a full Ethereum node is running
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {

cmd/geth/usage.go

@@ -25,6 +25,7 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/cmd/utils"
 	"github.com/EthereumCommonwealth/go-callisto/internal/debug"
 	"gopkg.in/urfave/cli.v1"
+	"strings"
 )
 
 // AppHelpTemplate is the test template for the default, global app help topic.
@@ -72,7 +73,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetworkIdFlag,
 			utils.TestnetFlag,
 			utils.RinkebyFlag,
-			utils.DevModeFlag,
 			utils.SyncModeFlag,
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
@@ -81,6 +81,12 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.LightKDFFlag,
 		},
 	},
+	{Name: "DEVELOPER CHAIN",
+		Flags: []cli.Flag{
+			utils.DeveloperFlag,
+			utils.DeveloperPeriodFlag,
+		},
+	},
 	{
 		Name: "ETHASH",
 		Flags: []cli.Flag{
@@ -92,6 +98,16 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.EthashDatasetsOnDiskFlag,
 		},
 	},
+	//{
+	//	Name: "DASHBOARD",
+	//	Flags: []cli.Flag{
+	//		utils.DashboardEnabledFlag,
+	//		utils.DashboardAddrFlag,
+	//		utils.DashboardPortFlag,
+	//		utils.DashboardRefreshFlag,
+	//		utils.DashboardAssetsFlag,
+	//	},
+	//},
 	{
 		Name: "TRANSACTION POOL",
 		Flags: []cli.Flag{
@@ -263,6 +279,9 @@ func init() {
 			uncategorized := []cli.Flag{}
 			for _, flag := range data.(*cli.App).Flags {
 				if _, ok := categorized[flag.String()]; !ok {
+					if strings.HasPrefix(flag.GetName(), "dashboard") {
+						continue
+					}
 					uncategorized = append(uncategorized, flag)
 				}
 			}

cmd/puppeth/genesis.go

@@ -0,0 +1,379 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"math"
+
+	"github.com/EthereumCommonwealth/go-callisto/common"
+	"github.com/EthereumCommonwealth/go-callisto/common/hexutil"
+	"github.com/EthereumCommonwealth/go-callisto/consensus/ethash"
+	"github.com/EthereumCommonwealth/go-callisto/core"
+	"github.com/EthereumCommonwealth/go-callisto/params"
+)
+
+// cppEthereumGenesisSpec represents the genesis specification format used by the
+// C++ Ethereum implementation.
+type cppEthereumGenesisSpec struct {
+	SealEngine string `json:"sealEngine"`
+	Params     struct {
+		AccountStartNonce       hexutil.Uint64 `json:"accountStartNonce"`
+		HomesteadForkBlock      hexutil.Uint64 `json:"homesteadForkBlock"`
+		EIP150ForkBlock         hexutil.Uint64 `json:"EIP150ForkBlock"`
+		EIP158ForkBlock         hexutil.Uint64 `json:"EIP158ForkBlock"`
+		ByzantiumForkBlock      hexutil.Uint64 `json:"byzantiumForkBlock"`
+		ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"`
+		NetworkID               hexutil.Uint64 `json:"networkID"`
+		ChainID                 hexutil.Uint64 `json:"chainID"`
+		MaximumExtraDataSize    hexutil.Uint64 `json:"maximumExtraDataSize"`
+		MinGasLimit             hexutil.Uint64 `json:"minGasLimit"`
+		MaxGasLimit             hexutil.Uint64 `json:"maxGasLimit"`
+		GasLimitBoundDivisor    *hexutil.Big   `json:"gasLimitBoundDivisor"`
+		MinimumDifficulty       *hexutil.Big   `json:"minimumDifficulty"`
+		DifficultyBoundDivisor  *hexutil.Big   `json:"difficultyBoundDivisor"`
+		DurationLimit           *hexutil.Big   `json:"durationLimit"`
+		BlockReward             *hexutil.Big   `json:"blockReward"`
+	} `json:"params"`
+
+	Genesis struct {
+		Nonce      hexutil.Bytes  `json:"nonce"`
+		Difficulty *hexutil.Big   `json:"difficulty"`
+		MixHash    common.Hash    `json:"mixHash"`
+		Author     common.Address `json:"author"`
+		Timestamp  hexutil.Uint64 `json:"timestamp"`
+		ParentHash common.Hash    `json:"parentHash"`
+		ExtraData  hexutil.Bytes  `json:"extraData"`
+		GasLimit   hexutil.Uint64 `json:"gasLimit"`
+	} `json:"genesis"`
+
+	Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"`
+}
+
+// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled
+// contract definition.
+type cppEthereumGenesisSpecAccount struct {
+	Balance     *hexutil.Big                   `json:"balance"`
+	Nonce       uint64                         `json:"nonce,omitempty"`
+	Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"`
+}
+
+// cppEthereumGenesisSpecBuiltin is the precompiled contract definition.
+type cppEthereumGenesisSpecBuiltin struct {
+	Name          string                               `json:"name,omitempty"`
+	StartingBlock hexutil.Uint64                       `json:"startingBlock,omitempty"`
+	Linear        *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"`
+}
+
+type cppEthereumGenesisSpecLinearPricing struct {
+	Base uint64 `json:"base"`
+	Word uint64 `json:"word"`
+}
+
+// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific
+// chain specification format.
+func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) {
+	// Only ethash is currently supported between go-ethereum and cpp-ethereum
+	if genesis.Config.Ethash == nil {
+		return nil, errors.New("unsupported consensus engine")
+	}
+	// Reconstruct the chain spec in Parity's format
+	spec := &cppEthereumGenesisSpec{
+		SealEngine: "Ethash",
+	}
+	spec.Params.AccountStartNonce = 0
+	spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64())
+	spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64())
+	spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64())
+	spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())
+	spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64)
+
+	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64())
+	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64())
+
+	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
+	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit.Uint64())
+	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64)
+	spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
+	spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (*hexutil.Big)(params.GasLimitBoundDivisor)
+	spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
+	spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
+
+	spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8))
+	binary.LittleEndian.PutUint64(spec.Genesis.Nonce[:], genesis.Nonce)
+
+	spec.Genesis.MixHash = genesis.Mixhash
+	spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty)
+	spec.Genesis.Author = genesis.Coinbase
+	spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp)
+	spec.Genesis.ParentHash = genesis.ParentHash
+	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
+	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
+
+	spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount)
+	for address, account := range genesis.Alloc {
+		spec.Accounts[address] = &cppEthereumGenesisSpecAccount{
+			Balance: (*hexutil.Big)(account.Balance),
+			Nonce:   account.Nonce,
+		}
+	}
+	spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+		Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+		Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+		Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+		Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3},
+	}
+	if genesis.Config.ByzantiumBlock != nil {
+		spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+		}
+		spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500},
+		}
+		spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000},
+		}
+		spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+		}
+	}
+	return spec, nil
+}
+
+// parityChainSpec is the chain specification format used by Parity.
+type parityChainSpec struct {
+	Name   string `json:"name"`
+	Engine struct {
+		Ethash struct {
+			Params struct {
+				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
+				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
+				GasLimitBoundDivisor   *hexutil.Big `json:"gasLimitBoundDivisor"`
+				DurationLimit          *hexutil.Big `json:"durationLimit"`
+				BlockReward            *hexutil.Big `json:"blockReward"`
+				HomesteadTransition    uint64       `json:"homesteadTransition"`
+				EIP150Transition       uint64       `json:"eip150Transition"`
+				EIP160Transition       uint64       `json:"eip160Transition"`
+				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
+				EIP161dTransition      uint64       `json:"eip161dTransition"`
+				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
+				EIP100bTransition      uint64       `json:"eip100bTransition"`
+				EIP649Transition       uint64       `json:"eip649Transition"`
+			} `json:"params"`
+		} `json:"Ethash"`
+	} `json:"engine"`
+
+	Params struct {
+		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
+		MinGasLimit          *hexutil.Big   `json:"minGasLimit"`
+		NetworkID            hexutil.Uint64 `json:"networkID"`
+		MaxCodeSize          uint64         `json:"maxCodeSize"`
+		EIP155Transition     uint64         `json:"eip155Transition"`
+		EIP98Transition      uint64         `json:"eip98Transition"`
+		EIP86Transition      uint64         `json:"eip86Transition"`
+		EIP140Transition     uint64         `json:"eip140Transition"`
+		EIP211Transition     uint64         `json:"eip211Transition"`
+		EIP214Transition     uint64         `json:"eip214Transition"`
+		EIP658Transition     uint64         `json:"eip658Transition"`
+	} `json:"params"`
+
+	Genesis struct {
+		Seal struct {
+			Ethereum struct {
+				Nonce   hexutil.Bytes `json:"nonce"`
+				MixHash hexutil.Bytes `json:"mixHash"`
+			} `json:"ethereum"`
+		} `json:"seal"`
+
+		Difficulty *hexutil.Big   `json:"difficulty"`
+		Author     common.Address `json:"author"`
+		Timestamp  hexutil.Uint64 `json:"timestamp"`
+		ParentHash common.Hash    `json:"parentHash"`
+		ExtraData  hexutil.Bytes  `json:"extraData"`
+		GasLimit   hexutil.Uint64 `json:"gasLimit"`
+	} `json:"genesis"`
+
+	Nodes    []string                                   `json:"nodes"`
+	Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"`
+}
+
+// parityChainSpecAccount is the prefunded genesis account and/or precompiled
+// contract definition.
+type parityChainSpecAccount struct {
+	Balance *hexutil.Big            `json:"balance"`
+	Nonce   uint64                  `json:"nonce,omitempty"`
+	Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"`
+}
+
+// parityChainSpecBuiltin is the precompiled contract definition.
+type parityChainSpecBuiltin struct {
+	Name       string                  `json:"name,omitempty"`
+	ActivateAt uint64                  `json:"activate_at,omitempty"`
+	Pricing    *parityChainSpecPricing `json:"pricing,omitempty"`
+}
+
+// parityChainSpecPricing represents the different pricing models that builtin
+// contracts might advertise using.
+type parityChainSpecPricing struct {
+	Linear       *parityChainSpecLinearPricing       `json:"linear,omitempty"`
+	ModExp       *parityChainSpecModExpPricing       `json:"modexp,omitempty"`
+	AltBnPairing *parityChainSpecAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"`
+}
+
+type parityChainSpecLinearPricing struct {
+	Base uint64 `json:"base"`
+	Word uint64 `json:"word"`
+}
+
+type parityChainSpecModExpPricing struct {
+	Divisor uint64 `json:"divisor"`
+}
+
+type parityChainSpecAltBnPairingPricing struct {
+	Base uint64 `json:"base"`
+	Pair uint64 `json:"pair"`
+}
+
+// newParityChainSpec converts a go-ethereum genesis block into a Parity specific
+// chain specification format.
+func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []string) (*parityChainSpec, error) {
+	// Only ethash is currently supported between go-ethereum and Parity
+	if genesis.Config.Ethash == nil {
+		return nil, errors.New("unsupported consensus engine")
+	}
+	// Reconstruct the chain spec in Parity's format
+	spec := &parityChainSpec{
+		Name:  network,
+		Nodes: bootnodes,
+	}
+	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
+	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
+	spec.Engine.Ethash.Params.GasLimitBoundDivisor = (*hexutil.Big)(params.GasLimitBoundDivisor)
+	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
+	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
+	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
+	spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64()
+	spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64()
+	spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64()
+	spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64()
+	spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward)
+	spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64()
+
+	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
+	spec.Params.MinGasLimit = (*hexutil.Big)(params.MinGasLimit)
+	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64())
+	spec.Params.MaxCodeSize = params.MaxCodeSize
+	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()
+	spec.Params.EIP98Transition = math.MaxUint64
+	spec.Params.EIP86Transition = math.MaxUint64
+	spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64()
+
+	spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8))
+	binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce)
+
+	spec.Genesis.Seal.Ethereum.MixHash = (hexutil.Bytes)(genesis.Mixhash[:])
+	spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty)
+	spec.Genesis.Author = genesis.Coinbase
+	spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp)
+	spec.Genesis.ParentHash = genesis.ParentHash
+	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
+	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
+
+	spec.Accounts = make(map[common.Address]*parityChainSpecAccount)
+	for address, account := range genesis.Alloc {
+		spec.Accounts[address] = &parityChainSpecAccount{
+			Balance: (*hexutil.Big)(account.Balance),
+			Nonce:   account.Nonce,
+		}
+	}
+	spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{
+		Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{
+		Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{
+		Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}},
+	}
+	spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{
+		Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}},
+	}
+	if genesis.Config.ByzantiumBlock != nil {
+		spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{
+			Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
+		}
+		spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{
+			Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
+		}
+		spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{
+			Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
+		}
+		spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{
+			Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
+		}
+	}
+	return spec, nil
+}
+
+// pyEthereumGenesisSpec represents the genesis specification format used by the
+// Python Ethereum implementation.
+type pyEthereumGenesisSpec struct {
+	Nonce      hexutil.Bytes     `json:"nonce"`
+	Timestamp  hexutil.Uint64    `json:"timestamp"`
+	ExtraData  hexutil.Bytes     `json:"extraData"`
+	GasLimit   hexutil.Uint64    `json:"gasLimit"`
+	Difficulty *hexutil.Big      `json:"difficulty"`
+	Mixhash    common.Hash       `json:"mixhash"`
+	Coinbase   common.Address    `json:"coinbase"`
+	Alloc      core.GenesisAlloc `json:"alloc"`
+	ParentHash common.Hash       `json:"parentHash"`
+}
+
+// newPyEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific
+// chain specification format.
+func newPyEthereumGenesisSpec(network string, genesis *core.Genesis) (*pyEthereumGenesisSpec, error) {
+	// Only ethash is currently supported between go-ethereum and pyethereum
+	if genesis.Config.Ethash == nil {
+		return nil, errors.New("unsupported consensus engine")
+	}
+	spec := &pyEthereumGenesisSpec{
+		Timestamp:  (hexutil.Uint64)(genesis.Timestamp),
+		ExtraData:  genesis.ExtraData,
+		GasLimit:   (hexutil.Uint64)(genesis.GasLimit),
+		Difficulty: (*hexutil.Big)(genesis.Difficulty),
+		Mixhash:    genesis.Mixhash,
+		Coinbase:   genesis.Coinbase,
+		Alloc:      genesis.Alloc,
+		ParentHash: genesis.ParentHash,
+	}
+	spec.Nonce = (hexutil.Bytes)(make([]byte, 8))
+	binary.LittleEndian.PutUint64(spec.Nonce[:], genesis.Nonce)
+
+	return spec, nil
+}

cmd/puppeth/module_dashboard.go

@@ -18,10 +18,12 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"math/rand"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
@@ -76,25 +78,26 @@ var dashboardContent = `
 						<div id="sidebar-menu" class="main_menu_side hidden-print main_menu">
 							<div class="menu_section">
 								<ul class="nav side-menu">
-									{{if .EthstatsPage}}<li><a onclick="load('//{{.EthstatsPage}}')"><i class="fa fa-tachometer"></i> Network Stats</a></li>{{end}}
+									{{if .EthstatsPage}}<li id="stats_menu"><a onclick="load('#stats')"><i class="fa fa-tachometer"></i> Network Stats</a></li>{{end}}
-									{{if .ExplorerPage}}<li><a onclick="load('//{{.ExplorerPage}}')"><i class="fa fa-database"></i> Block Explorer</a></li>{{end}}
+									{{if .ExplorerPage}}<li id="explorer_menu"><a onclick="load('#explorer')"><i class="fa fa-database"></i> Block Explorer</a></li>{{end}}
-									{{if .WalletPage}}<li><a onclick="load('//{{.WalletPage}}')"><i class="fa fa-address-book-o"></i> Browser Wallet</a></li>{{end}}
+									{{if .WalletPage}}<li id="wallet_menu"><a onclick="load('#wallet')"><i class="fa fa-address-book-o"></i> Browser Wallet</a></li>{{end}}
-									{{if .FaucetPage}}<li><a onclick="load('//{{.FaucetPage}}')"><i class="fa fa-bath"></i> Crypto Faucet</a></li>{{end}}
+									{{if .FaucetPage}}<li id="faucet_menu"><a onclick="load('#faucet')"><i class="fa fa-bath"></i> Crypto Faucet</a></li>{{end}}
-									<li id="connect"><a><i class="fa fa-plug"></i> Connect Yourself</a>
+									<li id="connect_menu"><a><i class="fa fa-plug"></i> Connect Yourself</a>
 										<ul id="connect_list" class="nav child_menu">
-											<li><a onclick="$('#connect').removeClass('active'); $('#connect_list').toggle(); load('#connect-go-ethereum-geth')">Go Ethereum: Geth</a></li>
+											<li><a onclick="$('#connect_menu').removeClass('active'); $('#connect_list').toggle(); load('#geth')">Go Ethereum: Geth</a></li>
-											<li><a onclick="$('#connect').removeClass('active'); $('#connect_list').toggle(); load('#connect-go-ethereum-mist')">Go Ethereum: Wallet & Mist</a></li>
+											<li><a onclick="$('#connect_menu').removeClass('active'); $('#connect_list').toggle(); load('#mist')">Go Ethereum: Wallet & Mist</a></li>
-											<li><a onclick="$('#connect').removeClass('active'); $('#connect_list').toggle(); load('#connect-go-ethereum-mobile')">Go Ethereum: Android & iOS</a></li>
+											<li><a onclick="$('#connect_menu').removeClass('active'); $('#connect_list').toggle(); load('#mobile')">Go Ethereum: Android & iOS</a></li>{{if .Ethash}}
+											<li><a onclick="$('#connect_menu').removeClass('active'); $('#connect_list').toggle(); load('#other')">Other Ethereum Clients</a></li>{{end}}
 										</ul>
 									</li>
-									<li><a onclick="load('#about')"><i class="fa fa-heartbeat"></i> About Puppeth</a></li>
+									<li id="about_menu"><a onclick="load('#about')"><i class="fa fa-heartbeat"></i> About Puppeth</a></li>
 								</ul>
 							</div>
 						</div>
 					</div>
 				</div>
-				<div class="right_col" role="main" style="padding: 0">
+				<div class="right_col" role="main" style="padding: 0 !important">
-					<div id="connect-go-ethereum-geth" hidden style="padding: 16px;">
+					<div id="geth" hidden style="padding: 16px;">
 						<div class="page-title">
 							<div class="title_left">
 								<h3>Connect Yourself &ndash; Go Ethereum: Geth</h3>
@@ -154,7 +157,7 @@ var dashboardContent = `
 										<p>Initial processing required to synchronize is light, as it only verifies the validity of the headers; similarly required disk capacity is small, tallying around 500 bytes per header. Low end machines with arbitrary storage, weak CPUs and 512MB+ RAM should cope well.</p>
 										<br/>
 										<p>To run a light node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
-											<pre>geth --datadir=$HOME/.{{.Network}} --light init {{.GethGenesis}}</pre>
+											<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
 											<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}</pre>
 										</p>
 										<br/>
@@ -173,8 +176,8 @@ var dashboardContent = `
 										<p>Initial processing required to synchronize is light, as it only verifies the validity of the headers; similarly required disk capacity is small, tallying around 500 bytes per header. Embedded machines with arbitrary storage, low power CPUs and 128MB+ RAM may work.</p>
 										<br/>
 										<p>To run an embedded node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
-											<pre>geth --datadir=$HOME/.{{.Network}} --light init {{.GethGenesis}}</pre>
+											<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
-											<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=32 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}</pre>
+											<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}</pre>
 										</p>
 										<br/>
 										<p>You can download Geth from <a href="https://geth.ethereum.org/downloads/" target="about:blank">https://geth.ethereum.org/downloads/</a>.</p>
@@ -183,7 +186,7 @@ var dashboardContent = `
 							</div>
 						</div>
 					</div>
-					<div id="connect-go-ethereum-mist" hidden style="padding: 16px;">
+					<div id="mist" hidden style="padding: 16px;">
 						<div class="page-title">
 							<div class="title_left">
 								<h3>Connect Yourself &ndash; Go Ethereum: Wallet &amp; Mist</h3>
@@ -235,7 +238,7 @@ var dashboardContent = `
 							</div>
 						</div>
 					</div>
-					<div id="connect-go-ethereum-mobile" hidden style="padding: 16px;">
+					<div id="mobile" hidden style="padding: 16px;">
 						<div class="page-title">
 							<div class="title_left">
 								<h3>Connect Yourself &ndash; Go Ethereum: Android &amp; iOS</h3>
@@ -309,7 +312,101 @@ try! node?.start();
 								</div>
 							</div>
 						</div>
-					</div>
+					</div>{{if .Ethash}}
+					<div id="other" hidden style="padding: 16px;">
+						<div class="page-title">
+							<div class="title_left">
+								<h3>Connect Yourself &ndash; Other Ethereum Clients</h3>
+							</div>
+						</div>
+						<div class="clearfix"></div>
+						<div class="row">
+							<div class="col-md-6">
+								<div class="x_panel">
+									<div class="x_title">
+										<h2>
+											<svg height="14px" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 115 115"><path fill="#5C8DBC" d="M9.7 83.3V35.5s0-3.4 3.3-5.2c3.3-1.8 39.6-23.5 39.6-23.5s4.6-3.1 9.4 0c0 0 43.1 23.9 42.4 25.3L85.3 43.3s-3.6-8.4-13.1-13c-11.3-5.5-29.7-6.2-42.9 13.3 0 0-8.6 13.5.3 31.6l-19 10.7s-.9-.6-.9-2.6z"/><path fill="#5C8DBC" d="M71 51.3c-2.8-4.7-7.9-7.9-13.8-7.9-8.8 0-16 7.2-16 16 0 2.8.7 5.4 2 7.7L71 51.3z"/><path fill="#194674" d="M43.1 67c2.8 4.7 7.9 7.9 13.8 7.9 8.8 0 16-7.2 16-16 0-2.8-.7-5.4-2-7.7L43.1 67z"/><path fill="#1B598E" d="M104.4 32.1s1.3 52.6-.3 53.6L58 58.6l46.4-26.5z"/><path fill="#FFF" d="M90 57h-3.9v-4.1h-4.2V57h-4v4.1h4V65h4.2v-3.9H90zm13.6 0h-3.9v-4.1h-4.2V57h-4v4.1h4V65h4.2v-3.9h3.9z"/><path fill="#194674" d="M29.5 75.1s9.2 17 28.5 16.1 27.3-16.6 27.3-16.6L104 85.4s4.1.8-41.6 25.7c0 0-4.9 3.3-10.2 0 0 0-41.3-23.1-41.6-25.3l18.9-10.7z"/></svg>
+											C++ Ethereum <small>Official C++ client from the Ethereum Foundation</small>
+										</h2>
+										<div class="clearfix"></div>
+									</div>
+									<div class="x_content">
+										<p>C++ Ethereum is the third most popular of the Ethereum clients, focusing on code portability to a broad range of operating systems and hardware. The client is currently a full node with transaction processing based synchronization.</p>
+										<br/>
+										<p>To run a cpp-ethereum node, download <a href="/{{.CppGenesis}}"><code>{{.CppGenesis}}</code></a> and start the node with:
+											<pre>eth --config {{.CppGenesis}} --datadir $HOME/.{{.Network}} --peerset "{{.CppBootnodes}}"</pre>
+										</p>
+										<br/>
+										<p>You can find cpp-ethereum at <a href="https://github.com/ethereum/cpp-ethereum/" target="about:blank">https://github.com/ethereum/cpp-ethereum/</a>.</p>
+									</div>
+								</div>
+							</div>
+							<div class="col-md-6">
+								<div class="x_panel">
+									<div class="x_title">
+										<h2>
+											<svg height="14px" version="1.1" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><path d="M46.42,13.07S24.51,18.54,35,30.6c3.09,3.55-.81,6.75-0.81,6.75s7.84-4,4.24-9.11C35,23.51,32.46,21.17,46.42,13.07ZM32.1,16.88C45.05,6.65,38.4,0,38.4,0c2.68,10.57-9.46,13.76-13.84,20.34-3,4.48,1.46,9.3,7.53,14.77C29.73,29.77,21.71,25.09,32.1,16.88Z" transform="translate(-8.4)" fill="#e57125"/><path d="M23.6,49.49c-9.84,2.75,6,8.43,18.51,3.06a23.06,23.06,0,0,1-3.52-1.72,36.62,36.62,0,0,1-13.25.56C21.16,50.92,23.6,49.49,23.6,49.49Zm17-5.36a51.7,51.7,0,0,1-17.1.82c-4.19-.43-1.45-2.46-1.45-2.46-10.84,3.6,6,7.68,21.18,3.25A7.59,7.59,0,0,1,40.62,44.13ZM51.55,54.68s1.81,1.49-2,2.64c-7.23,2.19-30.1,2.85-36.45.09-2.28-1,2-2.37,3.35-2.66a8.69,8.69,0,0,1,2.21-.25c-2.54-1.79-16.41,3.51-7,5C37.15,63.67,58.17,57.67,51.55,54.68ZM42.77,39.12a20.42,20.42,0,0,1,2.93-1.57s-4.83.86-9.65,1.27A87.37,87.37,0,0,1,20.66,39c-7.51-1,4.12-3.77,4.12-3.77A22,22,0,0,0,14.7,37.61C8.14,40.79,31,42.23,42.77,39.12Zm2.88,7.77a1,1,0,0,1-.24.31C61.44,43,55.54,32.35,47.88,35a2.19,2.19,0,0,0-1,.79,9,9,0,0,1,1.37-.37C52.1,34.66,57.65,40.65,45.64,46.89Zm0.43,14.75a94.76,94.76,0,0,1-29.17.45s1.47,1.22,9,1.7c11.53,0.74,29.22-.41,29.64-5.86C55.6,57.94,54.79,60,46.08,61.65Z" transform="translate(-8.4)" fill="#5482a2"/></svg>
+											Ethereum Harmony<small>Third party Java client from EtherCamp</small>
+										</h2>
+										<div class="clearfix"></div>
+									</div>
+									<div class="x_content">
+										<p>Ethereum Harmony is a web user-interface based graphical Ethereum client built on top of the EthereumJ Java implementation of the Ethereum protocol. The client currently is a full node with state download based synchronization.</p>
+										<br/>
+										<p>To run an Ethereum Harmony node, download <a href="/{{.HarmonyGenesis}}"><code>{{.HarmonyGenesis}}</code></a> and start the node with:
+											<pre>./gradlew runCustom -DgenesisFile={{.HarmonyGenesis}} -Dpeer.networkId={{.NetworkID}} -Ddatabase.dir=$HOME/.harmony/{{.Network}} {{.HarmonyBootnodes}} </pre>
+										</p>
+										<br/>
+										<p>You can find Ethereum Harmony at <a href="https://github.com/ether-camp/ethereum-harmony/" target="about:blank">https://github.com/ether-camp/ethereum-harmony/</a>.</p>
+									</div>
+								</div>
+							</div>
+						</div>
+						<div class="clearfix"></div>
+						<div class="row">
+							<div class="col-md-6">
+								<div class="x_panel">
+									<div class="x_title">
+										<h2>
+											<svg height="14px" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 104.56749 104.56675" version="1.1" viewbox="0 0 144 144" y="0px" x="0px"><metadata id="metadata10"><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs id="defs8" /><path style="fill:#676767;" id="path2" d="m 49.0125,12.3195 a 3.108,3.108 0 0 1 6.216,0 3.108,3.108 0 0 1 -6.216,0 m -37.077,28.14 a 3.108,3.108 0 0 1 6.216,0 3.108,3.108 0 0 1 -6.216,0 m 74.153,0.145 a 3.108,3.108 0 0 1 6.216,0 3.108,3.108 0 0 1 -6.216,0 m -65.156,4.258 c 1.43,-0.635 2.076,-2.311 1.441,-3.744 l -1.379,-3.118 h 5.423 v 24.444 h -10.941 a 38.265,38.265 0 0 1 -1.239,-14.607 z m 22.685,0.601 v -7.205 h 12.914 c 0.667,0 4.71,0.771 4.71,3.794 0,2.51 -3.101,3.41 -5.651,3.41 z m -17.631,38.793 a 3.108,3.108 0 0 1 6.216,0 3.108,3.108 0 0 1 -6.216,0 m 46.051,0.145 a 3.108,3.108 0 0 1 6.216,0 3.108,3.108 0 0 1 -6.216,0 m 0.961,-7.048 c -1.531,-0.328 -3.037,0.646 -3.365,2.18 l -1.56,7.28 a 38.265,38.265 0 0 1 -31.911,-0.153 l -1.559,-7.28 c -0.328,-1.532 -1.834,-2.508 -3.364,-2.179 l -6.427,1.38 a 38.265,38.265 0 0 1 -3.323,-3.917 h 31.272 c 0.354,0 0.59,-0.064 0.59,-0.386 v -11.062 c 0,-0.322 -0.236,-0.386 -0.59,-0.386 h -9.146 v -7.012 h 9.892 c 0.903,0 4.828,0.258 6.083,5.275 0.393,1.543 1.256,6.562 1.846,8.169 0.588,1.802 2.982,5.402 5.533,5.402 h 16.146 a 38.265,38.265 0 0 1 -3.544,4.102 z m 17.365,-29.207 a 38.265,38.265 0 0 1 0.081,6.643 h -3.926 c -0.393,0 -0.551,0.258 -0.551,0.643 v 1.803 c 0,4.244 -2.393,5.167 -4.49,5.402 -1.997,0.225 -4.211,-0.836 -4.484,-2.058 -1.178,-6.626 -3.141,-8.041 -6.241,-10.486 3.847,-2.443 7.85,-6.047 7.85,-10.871 0,-5.209 -3.571,-8.49 -6.005,-10.099 -3.415,-2.251 -7.196,-2.702 -8.216,-2.702 h -40.603 a 38.265,38.265 0 0 1 21.408,-12.082 l 4.786,5.021 c 1.082,1.133 2.874,1.175 4.006,0.092 l 5.355,-5.122 a 38.265,38.265 0 0 1 26.196,18.657 l -3.666,8.28 c -0.633,1.433 0.013,3.109 1.442,3.744 z m 9.143,0.134 -0.125,-1.28 3.776,-3.522 c 0.768,-0.716 0.481,-2.157 -0.501,-2.523 l -4.827,-1.805 -0.378,-1.246 3.011,-4.182 c 0.614,-0.85 0.05,-2.207 -0.984,-2.377 l -5.09,-0.828 -0.612,-1.143 2.139,-4.695 c 0.438,-0.956 -0.376,-2.179 -1.428,-2.139 l -5.166,0.18 -0.816,-0.99 1.187,-5.032 c 0.24,-1.022 -0.797,-2.06 -1.819,-1.82 l -5.031,1.186 -0.992,-0.816 0.181,-5.166 c 0.04,-1.046 -1.184,-1.863 -2.138,-1.429 l -4.694,2.14 -1.143,-0.613 -0.83,-5.091 c -0.168,-1.032 -1.526,-1.596 -2.376,-0.984 l -4.185,3.011 -1.244,-0.377 -1.805,-4.828 c -0.366,-0.984 -1.808,-1.267 -2.522,-0.503 l -3.522,3.779 -1.28,-0.125 -2.72,-4.395 c -0.55,-0.89 -2.023,-0.89 -2.571,0 l -2.72,4.395 -1.281,0.125 -3.523,-3.779 c -0.714,-0.764 -2.156,-0.481 -2.522,0.503 l -1.805,4.828 -1.245,0.377 -4.184,-3.011 c -0.85,-0.614 -2.209,-0.048 -2.377,0.984 l -0.83,5.091 -1.143,0.613 -4.694,-2.14 c -0.954,-0.436 -2.178,0.383 -2.138,1.429 l 0.18,5.166 -0.992,0.816 -5.031,-1.186 c -1.022,-0.238 -2.06,0.798 -1.82,1.82 l 1.185,5.032 -0.814,0.99 -5.166,-0.18 c -1.042,-0.03 -1.863,1.183 -1.429,2.139 l 2.14,4.695 -0.613,1.143 -5.09,0.828 c -1.034,0.168 -1.594,1.527 -0.984,2.377 l 3.011,4.182 -0.378,1.246 -4.828,1.805 c -0.98,0.366 -1.267,1.807 -0.501,2.523 l 3.777,3.522 -0.125,1.28 -4.394,2.72 c -0.89,0.55 -0.89,2.023 0,2.571 l 4.394,2.72 0.125,1.28 -3.777,3.523 c -0.766,0.714 -0.479,2.154 0.501,2.522 l 4.828,1.805 0.378,1.246 -3.011,4.183 c -0.612,0.852 -0.049,2.21 0.985,2.376 l 5.089,0.828 0.613,1.145 -2.14,4.693 c -0.436,0.954 0.387,2.181 1.429,2.139 l 5.164,-0.181 0.816,0.992 -1.185,5.033 c -0.24,1.02 0.798,2.056 1.82,1.816 l 5.031,-1.185 0.992,0.814 -0.18,5.167 c -0.04,1.046 1.184,1.864 2.138,1.428 l 4.694,-2.139 1.143,0.613 0.83,5.088 c 0.168,1.036 1.527,1.596 2.377,0.986 l 4.182,-3.013 1.246,0.379 1.805,4.826 c 0.366,0.98 1.808,1.269 2.522,0.501 l 3.523,-3.777 1.281,0.128 2.72,4.394 c 0.548,0.886 2.021,0.888 2.571,0 l 2.72,-4.394 1.28,-0.128 3.522,3.777 c 0.714,0.768 2.156,0.479 2.522,-0.501 l 1.805,-4.826 1.246,-0.379 4.183,3.013 c 0.85,0.61 2.208,0.048 2.376,-0.986 l 0.83,-5.088 1.143,-0.613 4.694,2.139 c 0.954,0.436 2.176,-0.38 2.138,-1.428 l -0.18,-5.167 0.991,-0.814 5.031,1.185 c 1.022,0.24 2.059,-0.796 1.819,-1.816 l -1.185,-5.033 0.814,-0.992 5.166,0.181 c 1.042,0.042 1.866,-1.185 1.428,-2.139 l -2.139,-4.693 0.612,-1.145 5.09,-0.828 c 1.036,-0.166 1.598,-1.524 0.984,-2.376 l -3.011,-4.183 0.378,-1.246 4.827,-1.805 c 0.982,-0.368 1.269,-1.808 0.501,-2.522 l -3.776,-3.523 0.125,-1.28 4.394,-2.72 c 0.89,-0.548 0.891,-2.021 10e-4,-2.571 z" /></svg>
+											Parity<small>Third party Rust client from Parity Technologies</small>
+										</h2>
+										<div class="clearfix"></div>
+									</div>
+									<div class="x_content">
+										<p>Parity is a fast, light and secure Ethereum client, supporting both headless mode of operation as well as a web user interface for direct manual interaction. The client is currently a full node with transaction processing based synchronization and state pruning enabled.</p>
+										<br/>
+										<p>To run a Parity node, download <a href="/{{.ParityGenesis}}"><code>{{.ParityGenesis}}</code></a> and start the node with:
+											<pre>parity --chain={{.ParityGenesis}}</pre>
+										</p>
+										<br/>
+										<p>You can find Parity at <a href="https://parity.io/" target="about:blank">https://parity.io/</a>.</p>
+									</div>
+								</div>
+							</div>
+							<div class="col-md-6">
+								<div class="x_panel">
+									<div class="x_title">
+										<h2>
+											<svg height="14px" version="1.1" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><defs><linearGradient id="a" x1="13.79" y1="38.21" x2="75.87" y2="-15.2" gradientTransform="matrix(0.56, 0, 0, -0.57, -8.96, 23.53)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#5c9fd3"/><stop offset="1" stop-color="#316a99"/></linearGradient><linearGradient id="b" x1="99.87" y1="-47.53" x2="77.7" y2="-16.16" gradientTransform="matrix(0.56, 0, 0, -0.57, -8.96, 23.53)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#ffd43d"/><stop offset="1" stop-color="#fee875"/></linearGradient></defs><g><path d="M31.62,0a43.6,43.6,0,0,0-7.3.62c-6.46,1.14-7.63,3.53-7.63,7.94v5.82H32v1.94H11a9.53,9.53,0,0,0-9.54,7.74,28.54,28.54,0,0,0,0,15.52c1.09,4.52,3.68,7.74,8.11,7.74h5.25v-7a9.7,9.7,0,0,1,9.54-9.48H39.58a7.69,7.69,0,0,0,7.63-7.76V8.56c0-4.14-3.49-7.25-7.63-7.94A47.62,47.62,0,0,0,31.62,0ZM23.37,4.68A2.91,2.91,0,1,1,20.5,7.6,2.9,2.9,0,0,1,23.37,4.68Z" transform="translate(-0.35)" fill="url(#a)"/><path d="M49.12,16.32V23.1a9.79,9.79,0,0,1-9.54,9.68H24.33a7.79,7.79,0,0,0-7.63,7.76V55.08c0,4.14,3.6,6.57,7.63,7.76a25.55,25.55,0,0,0,15.25,0c3.84-1.11,7.63-3.35,7.63-7.76V49.26H32V47.32H54.85c4.44,0,6.09-3.1,7.63-7.74s1.53-9.38,0-15.52c-1.1-4.42-3.19-7.74-7.63-7.74H49.12ZM40.54,53.14A2.91,2.91,0,1,1,37.67,56,2.88,2.88,0,0,1,40.54,53.14Z" transform="translate(-0.35)" fill="url(#b)"/></g></svg>
+											PyEthApp<small>Official Python client from the Ethereum Foundation</small>
+										</h2>
+										<div class="clearfix"></div>
+									</div>
+									<div class="x_content">
+										<p>Pyethapp is the Ethereum Foundation's research client, aiming to provide an easily hackable and extendable codebase. The client is currently a full node with transaction processing based synchronization and state pruning enabled.</p>
+										<br/>
+										<p>To run a pyethapp node, download <a href="/{{.PythonGenesis}}"><code>{{.PythonGenesis}}</code></a> and start the node with:
+											<pre>mkdir -p $HOME/.config/pyethapp/{{.Network}}</pre>
+											<pre>pyethapp -c eth.genesis="$(cat {{.PythonGenesis}})" -c eth.network_id={{.NetworkID}} -c data_dir=$HOME/.config/pyethapp/{{.Network}} -c discovery.bootstrap_nodes="[{{.PythonBootnodes}}]" -c eth.block.HOMESTEAD_FORK_BLKNUM={{.Homestead}} -c eth.block.ANTI_DOS_FORK_BLKNUM={{.Tangerine}} -c eth.block.SPURIOUS_DRAGON_FORK_BLKNUM={{.Spurious}} -c eth.block.METROPOLIS_FORK_BLKNUM={{.Byzantium}} -c eth.block.DAO_FORK_BLKNUM=18446744073709551615 run --console</pre>
+										</p>
+										<br/>
+										<p>You can find pyethapp at <a href="https://github.com/ethereum/pyethapp/" target="about:blank">https://github.com/ethereum/pyethapp/</a>.</p>
+									</div>
+								</div>
+							</div>
+						</div>
+					</div>{{end}}
 					<div id="about" hidden>
 						<div class="row vertical-center">
 							<div style="margin: 0 auto;">
@@ -344,13 +441,33 @@ try! node?.start();
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
 		<script src="https://cdnjs.cloudflare.com/ajax/libs/gentelella/1.3.0/js/custom.min.js"></script>
 		<script>
-			var load = function(url) {
+			var load = function(hash) {
-				$("#connect-go-ethereum-geth").fadeOut(300)
+				window.location.hash = hash;
-				$("#connect-go-ethereum-mist").fadeOut(300)
+
-				$("#connect-go-ethereum-mobile").fadeOut(300)
+				// Fade out all possible pages (yes, ugly, no, don't care)
+				$("#geth").fadeOut(300)
+				$("#mist").fadeOut(300)
+				$("#mobile").fadeOut(300)
+				$("#other").fadeOut(300)
 				$("#about").fadeOut(300)
 				$("#frame-wrapper").fadeOut(300);
 
+				// Depending on the hash, resolve it into a local or remote URL
+				var url = hash;
+				switch (hash) {
+					case "#stats":
+						url = "//{{.EthstatsPage}}";
+						break;
+					case "#explorer":
+						url = "//{{.ExplorerPage}}";
+						break;
+					case "#wallet":
+						url = "//{{.WalletPage}}";
+						break;
+					case "#faucet":
+						url = "//{{.FaucetPage}}";
+						break;
+				}
 				setTimeout(function() {
 					if (url.substring(0, 1) == "#") {
 						$('.body').css({overflowY: 'auto'});
@@ -364,13 +481,10 @@ try! node?.start();
 			}
 			var resize = function() {
 				var sidebar = $($(".navbar")[0]).width();
-				var content = 1920;
 				var limit   = document.body.clientWidth - sidebar;
-				var scale   = limit / content;
+				var scale   = limit / 1920;
 
-				console.log(document.body.clientHeight);
+				$("#frame-wrapper").width(limit);
-
-				$("#frame-wrapper").width(content / scale);
 				$("#frame-wrapper").height(document.body.clientHeight / scale);
 				$("#frame-wrapper").css({
 					transform: 'scale(' + (scale) + ')',
@@ -379,9 +493,17 @@ try! node?.start();
 			};
 			$(window).resize(resize);
 
-			var item = $(".side-menu").children()[0];
+			if (window.location.hash == "") {
-			$(item).children()[0].click();
+				var item = $(".side-menu").children()[0];
-			$(item).addClass("active");
+				$(item).children()[0].click();
+				$(item).addClass("active");
+			} else {
+				load(window.location.hash);
+				var menu = $(window.location.hash + "_menu");
+				if (menu !== undefined) {
+					$(menu).addClass("active");
+				}
+			}
 		</script>
 	</body>
 </html>
@@ -405,6 +527,10 @@ RUN \
 	echo '});'                                                             >> server.js
 
 ADD {{.Network}}.json /dashboard/{{.Network}}.json
+ADD {{.Network}}-cpp.json /dashboard/{{.Network}}-cpp.json
+ADD {{.Network}}-harmony.json /dashboard/{{.Network}}-harmony.json
+ADD {{.Network}}-parity.json /dashboard/{{.Network}}-parity.json
+ADD {{.Network}}-python.json /dashboard/{{.Network}}-python.json
 ADD index.html /dashboard/index.html
 ADD puppeth.png /dashboard/puppeth.png
 
@@ -422,8 +548,12 @@ services:
     build: .
     image: {{.Network}}/dashboard{{if not .VHost}}
     ports:
-      - "{{.Port}}:80"{{else}}
+      - "{{.Port}}:80"{{end}}
     environment:
+      - ETHSTATS_PAGE={{.EthstatsPage}}
+      - EXPLORER_PAGE={{.ExplorerPage}}
+      - WALLET_PAGE={{.WalletPage}}
+      - FAUCET_PAGE={{.FaucetPage}}{{if .VHost}}
       - VIRTUAL_HOST={{.VHost}}{{end}}
     logging:
       driver: "json-file"
@@ -436,7 +566,7 @@ services:
 // deployDashboard deploys a new dashboard container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployDashboard(client *sshClient, network string, port int, vhost string, services map[string]string, conf *config, ethstats bool) ([]byte, error) {
+func deployDashboard(client *sshClient, network string, conf *config, config *dashboardInfos, nocache bool) ([]byte, error) {
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
 	files := make(map[string][]byte)
@@ -449,37 +579,95 @@ func deployDashboard(client *sshClient, network string, port int, vhost string,
 
 	composefile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(dashboardComposefile)).Execute(composefile, map[string]interface{}{
-		"Network": network,
+		"Network":      network,
-		"Port":    port,
+		"Port":         config.port,
-		"VHost":   vhost,
+		"VHost":        config.host,
+		"EthstatsPage": config.ethstats,
+		"ExplorerPage": config.explorer,
+		"WalletPage":   config.wallet,
+		"FaucetPage":   config.faucet,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
 	statsLogin := fmt.Sprintf("yournode:%s", conf.ethstats)
-	if !ethstats {
+	if !config.trusted {
 		statsLogin = ""
 	}
 	indexfile := new(bytes.Buffer)
+	bootCpp := make([]string, len(conf.bootFull))
+	for i, boot := range conf.bootFull {
+		bootCpp[i] = "required:" + strings.TrimPrefix(boot, "enode://")
+	}
+	bootHarmony := make([]string, len(conf.bootFull))
+	for i, boot := range conf.bootFull {
+		bootHarmony[i] = fmt.Sprintf("-Dpeer.active.%d.url=%s", i, boot)
+	}
+	bootPython := make([]string, len(conf.bootFull))
+	for i, boot := range conf.bootFull {
+		bootPython[i] = "'" + boot + "'"
+	}
 	template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{
 		"Network":            network,
-		"NetworkID":          conf.genesis.Config.ChainId,
+		"NetworkID":          conf.Genesis.Config.ChainId,
 		"NetworkTitle":       strings.Title(network),
-		"EthstatsPage":       services["ethstats"],
+		"EthstatsPage":       config.ethstats,
-		"ExplorerPage":       services["explorer"],
+		"ExplorerPage":       config.explorer,
-		"WalletPage":         services["wallet"],
+		"WalletPage":         config.wallet,
-		"FaucetPage":         services["faucet"],
+		"FaucetPage":         config.faucet,
 		"GethGenesis":        network + ".json",
 		"BootnodesFull":      conf.bootFull,
 		"BootnodesLight":     conf.bootLight,
 		"BootnodesFullFlat":  strings.Join(conf.bootFull, ","),
 		"BootnodesLightFlat": strings.Join(conf.bootLight, ","),
 		"Ethstats":           statsLogin,
+		"Ethash":             conf.Genesis.Config.Ethash != nil,
+		"CppGenesis":         network + "-cpp.json",
+		"CppBootnodes":       strings.Join(bootCpp, " "),
+		"HarmonyGenesis":     network + "-harmony.json",
+		"HarmonyBootnodes":   strings.Join(bootHarmony, " "),
+		"ParityGenesis":      network + "-parity.json",
+		"PythonGenesis":      network + "-python.json",
+		"PythonBootnodes":    strings.Join(bootPython, ","),
+		"Homestead":          conf.Genesis.Config.HomesteadBlock,
+		"Tangerine":          conf.Genesis.Config.EIP150Block,
+		"Spurious":           conf.Genesis.Config.EIP155Block,
+		"Byzantium":          conf.Genesis.Config.ByzantiumBlock,
 	})
 	files[filepath.Join(workdir, "index.html")] = indexfile.Bytes()
 
-	genesis, _ := conf.genesis.MarshalJSON()
+	// Marshal the genesis spec files for go-ethereum and all the other clients
+	genesis, _ := conf.Genesis.MarshalJSON()
 	files[filepath.Join(workdir, network+".json")] = genesis
 
+	if conf.Genesis.Config.Ethash != nil {
+		cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis)
+		if err != nil {
+			return nil, err
+		}
+		cppSpecJSON, _ := json.Marshal(cppSpec)
+		files[filepath.Join(workdir, network+"-cpp.json")] = cppSpecJSON
+
+		harmonySpecJSON, _ := conf.Genesis.MarshalJSON()
+		files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON
+
+		paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootFull)
+		if err != nil {
+			return nil, err
+		}
+		paritySpecJSON, _ := json.Marshal(paritySpec)
+		files[filepath.Join(workdir, network+"-parity.json")] = paritySpecJSON
+
+		pyethSpec, err := newPyEthereumGenesisSpec(network, conf.Genesis)
+		if err != nil {
+			return nil, err
+		}
+		pyethSpecJSON, _ := json.Marshal(pyethSpec)
+		files[filepath.Join(workdir, network+"-python.json")] = pyethSpecJSON
+	} else {
+		for _, client := range []string{"cpp", "harmony", "parity", "python"} {
+			files[filepath.Join(workdir, network+"-"+client+".json")] = []byte{}
+		}
+	}
 	files[filepath.Join(workdir, "puppeth.png")] = dashboardMascot
 
 	// Upload the deployment files to the remote server (and clean up afterwards)
@@ -489,19 +677,36 @@ func deployDashboard(client *sshClient, network string, port int, vhost string,
 	defer client.Run("rm -rf " + workdir)
 
 	// Build and deploy the dashboard service
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
 // dashboardInfos is returned from an dashboard status check to allow reporting
 // various configuration parameters.
 type dashboardInfos struct {
-	host string
+	host    string
-	port int
+	port    int
+	trusted bool
+
+	ethstats string
+	explorer string
+	wallet   string
+	faucet   string
 }
 
-// String implements the stringer interface.
+// Report converts the typed struct into a plain string->string map, containing
-func (info *dashboardInfos) String() string {
+// most - but not all - fields for reporting to the user.
-	return fmt.Sprintf("host=%s, port=%d", info.host, info.port)
+func (info *dashboardInfos) Report() map[string]string {
+	return map[string]string{
+		"Website address":       info.host,
+		"Website listener port": strconv.Itoa(info.port),
+		"Ethstats service":      info.ethstats,
+		"Explorer service":      info.explorer,
+		"Wallet service":        info.wallet,
+		"Faucet service":        info.faucet,
+	}
 }
 
 // checkDashboard does a health-check against a dashboard container to verify if
@@ -536,7 +741,11 @@ func checkDashboard(client *sshClient, network string) (*dashboardInfos, error)
 	}
 	// Container available, assemble and return the useful infos
 	return &dashboardInfos{
-		host: host,
+		host:     host,
-		port: port,
+		port:     port,
+		ethstats: infos.envvars["ETHSTATS_PAGE"],
+		explorer: infos.envvars["EXPLORER_PAGE"],
+		wallet:   infos.envvars["WALLET_PAGE"],
+		faucet:   infos.envvars["FAUCET_PAGE"],
 	}, nil
 }

cmd/puppeth/module_ethstats.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math/rand"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
 
@@ -30,21 +31,9 @@ import (
 // ethstatsDockerfile is the Dockerfile required to build an ethstats backend
 // and associated monitoring site.
 var ethstatsDockerfile = `
-FROM mhart/alpine-node:latest
+FROM puppeth/ethstats:latest
-
-RUN \
-  apk add --update git                                         && \
-  git clone --depth=1 https://github.com/karalabe/eth-netstats && \
-	apk del git && rm -rf /var/cache/apk/*                       && \
-	\
-  cd /eth-netstats && npm install && npm install -g grunt-cli && grunt
-
-WORKDIR /eth-netstats
-EXPOSE 3000
 
 RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}], reserved: ["yournode"]};' > lib/utils/config.js
-
-CMD ["npm", "start"]
 `
 
 // ethstatsComposefile is the docker-compose.yml file required to deploy and
@@ -72,7 +61,7 @@ services:
 // deployEthstats deploys a new ethstats container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string, banned []string) ([]byte, error) {
+func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string, banned []string, nocache bool) ([]byte, error) {
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
 	files := make(map[string][]byte)
@@ -110,7 +99,10 @@ func deployEthstats(client *sshClient, network string, port int, secret string,
 	defer client.Run("rm -rf " + workdir)
 
 	// Build and deploy the ethstats service
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
 // ethstatsInfos is returned from an ethstats status check to allow reporting
@@ -123,9 +115,15 @@ type ethstatsInfos struct {
 	banned []string
 }
 
-// String implements the stringer interface.
+// Report converts the typed struct into a plain string->string map, containing
-func (info *ethstatsInfos) String() string {
+// most - but not all - fields for reporting to the user.
-	return fmt.Sprintf("host=%s, port=%d, secret=%s, banned=%v", info.host, info.port, info.secret, info.banned)
+func (info *ethstatsInfos) Report() map[string]string {
+	return map[string]string{
+		"Website address":       info.host,
+		"Website listener port": strconv.Itoa(info.port),
+		"Login secret":          info.secret,
+		"Banned addresses":      fmt.Sprintf("%v", info.banned),
+	}
 }
 
 // checkEthstats does a health-check against an ethstats server to verify whether

cmd/puppeth/module_explorer.go

@@ -0,0 +1,211 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+)
+
+// explorerDockerfile is the Dockerfile required to run a block explorer.
+var explorerDockerfile = `
+FROM puppeth/explorer:latest
+
+ADD ethstats.json /ethstats.json
+ADD chain.json /chain.json
+
+RUN \
+  echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' >  explorer.sh && \
+	echo '(cd ../etherchain-light && npm start &)'                      >> explorer.sh && \
+	echo '/parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
+
+ENTRYPOINT ["/bin/sh", "explorer.sh"]
+`
+
+// explorerEthstats is the configuration file for the ethstats javascript client.
+var explorerEthstats = `[
+  {
+    "name"              : "node-app",
+    "script"            : "app.js",
+    "log_date_format"   : "YYYY-MM-DD HH:mm Z",
+    "merge_logs"        : false,
+    "watch"             : false,
+    "max_restarts"      : 10,
+    "exec_interpreter"  : "node",
+    "exec_mode"         : "fork_mode",
+    "env":
+    {
+      "NODE_ENV"        : "production",
+      "RPC_HOST"        : "localhost",
+      "RPC_PORT"        : "8545",
+      "LISTENING_PORT"  : "{{.Port}}",
+      "INSTANCE_NAME"   : "{{.Name}}",
+      "CONTACT_DETAILS" : "",
+      "WS_SERVER"       : "{{.Host}}",
+      "WS_SECRET"       : "{{.Secret}}",
+      "VERBOSITY"       : 2
+    }
+  }
+]`
+
+// explorerComposefile is the docker-compose.yml file required to deploy and
+// maintain a block explorer.
+var explorerComposefile = `
+version: '2'
+services:
+  explorer:
+    build: .
+    image: {{.Network}}/explorer
+    ports:
+      - "{{.NodePort}}:{{.NodePort}}"
+      - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}}
+      - "{{.WebPort}}:3000"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.local/share/io.parity.ethereum
+    environment:
+      - NODE_PORT={{.NodePort}}/tcp
+      - STATS={{.Ethstats}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}
+      - VIRTUAL_PORT=3000{{end}}
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1m"
+        max-file: "10"
+    restart: always
+`
+
+// deployExplorer deploys a new block explorer container to a remote machine via
+// SSH, docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployExplorer(client *sshClient, network string, chainspec []byte, config *explorerInfos, nocache bool) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(explorerDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"NodePort": config.nodePort,
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	ethstats := new(bytes.Buffer)
+	template.Must(template.New("").Parse(explorerEthstats)).Execute(ethstats, map[string]interface{}{
+		"Port":   config.nodePort,
+		"Name":   config.ethstats[:strings.Index(config.ethstats, ":")],
+		"Secret": config.ethstats[strings.Index(config.ethstats, ":")+1 : strings.Index(config.ethstats, "@")],
+		"Host":   config.ethstats[strings.Index(config.ethstats, "@")+1:],
+	})
+	files[filepath.Join(workdir, "ethstats.json")] = ethstats.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(explorerComposefile)).Execute(composefile, map[string]interface{}{
+		"Datadir":  config.datadir,
+		"Network":  network,
+		"NodePort": config.nodePort,
+		"VHost":    config.webHost,
+		"WebPort":  config.webPort,
+		"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	files[filepath.Join(workdir, "chain.json")] = chainspec
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the boot or seal node service
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+}
+
+// explorerInfos is returned from a block explorer status check to allow reporting
+// various configuration parameters.
+type explorerInfos struct {
+	datadir  string
+	ethstats string
+	nodePort int
+	webHost  string
+	webPort  int
+}
+
+// Report converts the typed struct into a plain string->string map, containing
+// most - but not all - fields for reporting to the user.
+func (info *explorerInfos) Report() map[string]string {
+	report := map[string]string{
+		"Data directory":         info.datadir,
+		"Node listener port ":    strconv.Itoa(info.nodePort),
+		"Ethstats username":      info.ethstats,
+		"Website address ":       info.webHost,
+		"Website listener port ": strconv.Itoa(info.webPort),
+	}
+	return report
+}
+
+// checkExplorer does a health-check against an block explorer server to verify
+// whether it's running, and if yes, whether it's responsive.
+func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
+	// Inspect a possible block explorer container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_explorer_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	webPort := infos.portmap["3000/tcp"]
+	if webPort == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			webPort = proxy.port
+		}
+	}
+	if webPort == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and the config values
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	// Run a sanity check to see if the devp2p is reachable
+	nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
+	if err = checkPort(client.server, nodePort); err != nil {
+		log.Warn(fmt.Sprintf("Explorer devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
+	}
+	// Assemble and return the useful infos
+	stats := &explorerInfos{
+		datadir:  infos.volumes["/root/.local/share/io.parity.ethereum"],
+		nodePort: nodePort,
+		webHost:  host,
+		webPort:  webPort,
+		ethstats: infos.envvars["STATS"],
+	}
+	return stats, nil
+}

cmd/puppeth/module_faucet.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"math/rand"
@@ -25,36 +26,24 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/log"
 )
 
 // faucetDockerfile is the Dockerfile required to build an faucet container to
 // grant crypto tokens based on GitHub authentications.
 var faucetDockerfile = `
-FROM alpine:latest
+FROM ethereum/client-go:alltools-latest
-
-RUN mkdir /go
-ENV GOPATH /go
-
-RUN \
-  apk add --update git go make gcc musl-dev ca-certificates linux-headers                             && \
-	mkdir -p $GOPATH/src/github.com/ethereum                                                            && \
-	(cd $GOPATH/src/github.com/ethereum && git clone --depth=1 https://github.com/EthereumCommonwealth/go-callisto) && \
-  go build -v github.com/EthereumCommonwealth/go-callisto/cmd/faucet                                              && \
-  apk del git go make gcc musl-dev linux-headers                                                      && \
-  rm -rf $GOPATH && rm -rf /var/cache/apk/*
 
 ADD genesis.json /genesis.json
 ADD account.json /account.json
 ADD account.pass /account.pass
 
-EXPOSE 8080
+ENTRYPOINT [ \
-
+	"faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", "--ethport", "{{.EthPort}}",     \
-CMD [ \
+	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",             \
-	"/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", "--ethport", "{{.EthPort}}", \
+	"--account.json", "/account.json", "--account.pass", "/account.pass"                                                                                                    \
-	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",          \
+	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}}                          \
-	"--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass"                       \
-	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}                                                        \
 ]`
 
 // faucetComposefile is the docker-compose.yml file required to deploy and maintain
@@ -76,10 +65,9 @@ services:
       - FAUCET_AMOUNT={{.FaucetAmount}}
       - FAUCET_MINUTES={{.FaucetMinutes}}
       - FAUCET_TIERS={{.FaucetTiers}}
-      - GITHUB_USER={{.GitHubUser}}
-      - GITHUB_TOKEN={{.GitHubToken}}
       - CAPTCHA_TOKEN={{.CaptchaToken}}
-      - CAPTCHA_SECRET={{.CaptchaSecret}}{{if .VHost}}
+      - CAPTCHA_SECRET={{.CaptchaSecret}}
+      - NO_AUTH={{.NoAuth}}{{if .VHost}}
       - VIRTUAL_HOST={{.VHost}}
       - VIRTUAL_PORT=8080{{end}}
     logging:
@@ -93,7 +81,7 @@ services:
 // deployFaucet deploys a new faucet container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployFaucet(client *sshClient, network string, bootnodes []string, config *faucetInfos) ([]byte, error) {
+func deployFaucet(client *sshClient, network string, bootnodes []string, config *faucetInfos, nocache bool) ([]byte, error) {
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
 	files := make(map[string][]byte)
@@ -104,14 +92,13 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"Bootnodes":     strings.Join(bootnodes, ","),
 		"Ethstats":      config.node.ethstats,
 		"EthPort":       config.node.portFull,
-		"GitHubUser":    config.githubUser,
-		"GitHubToken":   config.githubToken,
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
 		"FaucetName":    strings.Title(network),
 		"FaucetAmount":  config.amount,
 		"FaucetMinutes": config.minutes,
 		"FaucetTiers":   config.tiers,
+		"NoAuth":        config.noauth,
 	})
 	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
 
@@ -123,17 +110,16 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"ApiPort":       config.port,
 		"EthPort":       config.node.portFull,
 		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
-		"GitHubUser":    config.githubUser,
-		"GitHubToken":   config.githubToken,
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
 		"FaucetAmount":  config.amount,
 		"FaucetMinutes": config.minutes,
 		"FaucetTiers":   config.tiers,
+		"NoAuth":        config.noauth,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
-	files[filepath.Join(workdir, "genesis.json")] = []byte(config.node.genesis)
+	files[filepath.Join(workdir, "genesis.json")] = config.node.genesis
 	files[filepath.Join(workdir, "account.json")] = []byte(config.node.keyJSON)
 	files[filepath.Join(workdir, "account.pass")] = []byte(config.node.keyPass)
 
@@ -144,7 +130,10 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 	defer client.Run("rm -rf " + workdir)
 
 	// Build and deploy the faucet service
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
 // faucetInfos is returned from an faucet status check to allow reporting various
@@ -156,15 +145,38 @@ type faucetInfos struct {
 	amount        int
 	minutes       int
 	tiers         int
-	githubUser    string
+	noauth        bool
-	githubToken   string
 	captchaToken  string
 	captchaSecret string
 }
 
-// String implements the stringer interface.
+// Report converts the typed struct into a plain string->string map, containing
-func (info *faucetInfos) String() string {
+// most - but not all - fields for reporting to the user.
-	return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, tiers=%d, github=%s, captcha=%v, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.tiers, info.githubUser, info.captchaToken != "", info.node.ethstats)
+func (info *faucetInfos) Report() map[string]string {
+	report := map[string]string{
+		"Website address":              info.host,
+		"Website listener port":        strconv.Itoa(info.port),
+		"Ethereum listener port":       strconv.Itoa(info.node.portFull),
+		"Funding amount (base tier)":   fmt.Sprintf("%d Ethers", info.amount),
+		"Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes),
+		"Funding tiers":                strconv.Itoa(info.tiers),
+		"Captha protection":            fmt.Sprintf("%v", info.captchaToken != ""),
+		"Ethstats username":            info.node.ethstats,
+	}
+	if info.noauth {
+		report["Debug mode (no auth)"] = "enabled"
+	}
+	if info.node.keyJSON != "" {
+		var key struct {
+			Address string `json:"address"`
+		}
+		if err := json.Unmarshal([]byte(info.node.keyJSON), &key); err == nil {
+			report["Funding account"] = common.HexToAddress(key.Address).Hex()
+		} else {
+			log.Error("Failed to retrieve signer address", "err", err)
+		}
+	}
+	return report
 }
 
 // checkFaucet does a health-check against an faucet server to verify whether
@@ -224,9 +236,8 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
 		amount:        amount,
 		minutes:       minutes,
 		tiers:         tiers,
-		githubUser:    infos.envvars["GITHUB_USER"],
-		githubToken:   infos.envvars["GITHUB_TOKEN"],
 		captchaToken:  infos.envvars["CAPTCHA_TOKEN"],
 		captchaSecret: infos.envvars["CAPTCHA_SECRET"],
+		noauth:        infos.envvars["NO_AUTH"] == "true",
 	}, nil
 }

cmd/puppeth/module_nginx.go

@@ -22,6 +22,7 @@ import (
 	"html/template"
 	"math/rand"
 	"path/filepath"
+	"strconv"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
 )
@@ -54,7 +55,7 @@ services:
 // deployNginx deploys a new nginx reverse-proxy container to expose one or more
 // HTTP services running on a single host. If an instance with the specified
 // network name already exists there, it will be overwritten!
-func deployNginx(client *sshClient, network string, port int) ([]byte, error) {
+func deployNginx(client *sshClient, network string, port int, nocache bool) ([]byte, error) {
 	log.Info("Deploying nginx reverse-proxy", "server", client.server, "port", port)
 
 	// Generate the content to upload to the server
@@ -78,8 +79,11 @@ func deployNginx(client *sshClient, network string, port int) ([]byte, error) {
 	}
 	defer client.Run("rm -rf " + workdir)
 
-	// Build and deploy the ethstats service
+	// Build and deploy the reverse-proxy service
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
 // nginxInfos is returned from an nginx reverse-proxy status check to allow
@@ -88,9 +92,12 @@ type nginxInfos struct {
 	port int
 }
 
-// String implements the stringer interface.
+// Report converts the typed struct into a plain string->string map, containing
-func (info *nginxInfos) String() string {
+// most - but not all - fields for reporting to the user.
-	return fmt.Sprintf("port=%d", info.port)
+func (info *nginxInfos) Report() map[string]string {
+	return map[string]string{
+		"Shared listener port": strconv.Itoa(info.port),
+	}
 }
 
 // checkNginx does a health-check against an nginx reverse-proxy to verify whether

cmd/puppeth/module_node.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"path/filepath"
@@ -25,6 +26,7 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/log"
 )
 
@@ -38,9 +40,9 @@ ADD genesis.json /genesis.json
 	ADD signer.pass /signer.pass
 {{end}}
 RUN \
-  echo 'geth init /genesis.json' > geth.sh && \{{if .Unlock}}
+  echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -58,7 +60,8 @@ services:
       - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
       - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
     volumes:
-      - {{.Datadir}}:/root/.ethereum
+      - {{.Datadir}}:/root/.ethereum{{if .Ethashdir}}
+      - {{.Ethashdir}}:/root/.ethash{{end}}
     environment:
       - FULL_PORT={{.FullPort}}/tcp
       - LIGHT_PORT={{.LightPort}}/udp
@@ -79,7 +82,7 @@ services:
 // deployNode deploys a new Ethereum node container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos) ([]byte, error) {
+func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos, nocache bool) ([]byte, error) {
 	kind := "sealnode"
 	if config.keyJSON == "" && config.etherbase == "" {
 		kind = "bootnode"
@@ -114,6 +117,7 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	template.Must(template.New("").Parse(nodeComposefile)).Execute(composefile, map[string]interface{}{
 		"Type":       kind,
 		"Datadir":    config.datadir,
+		"Ethashdir":  config.ethashdir,
 		"Network":    network,
 		"FullPort":   config.portFull,
 		"TotalPeers": config.peersTotal,
@@ -127,9 +131,7 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
-	//genesisfile, _ := json.MarshalIndent(config.genesis, "", "  ")
+	files[filepath.Join(workdir, "genesis.json")] = config.genesis
-	files[filepath.Join(workdir, "genesis.json")] = []byte(config.genesis)
-
 	if config.keyJSON != "" {
 		files[filepath.Join(workdir, "signer.json")] = []byte(config.keyJSON)
 		files[filepath.Join(workdir, "signer.pass")] = []byte(config.keyPass)
@@ -141,7 +143,10 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	defer client.Run("rm -rf " + workdir)
 
 	// Build and deploy the boot or seal node service
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
 }
 
 // nodeInfos is returned from a boot or seal node status check to allow reporting
@@ -150,6 +155,7 @@ type nodeInfos struct {
 	genesis    []byte
 	network    int64
 	datadir    string
+	ethashdir  string
 	ethstats   string
 	portFull   int
 	portLight  int
@@ -164,14 +170,43 @@ type nodeInfos struct {
 	gasPrice   float64
 }
 
-// String implements the stringer interface.
+// Report converts the typed struct into a plain string->string map, containing
-func (info *nodeInfos) String() string {
+// most - but not all - fields for reporting to the user.
-	discv5 := ""
+func (info *nodeInfos) Report() map[string]string {
-	if info.peersLight > 0 {
+	report := map[string]string{
-		discv5 = fmt.Sprintf(", portv5=%d", info.portLight)
+		"Data directory":             info.datadir,
+		"Listener port (full nodes)": strconv.Itoa(info.portFull),
+		"Peer count (all total)":     strconv.Itoa(info.peersTotal),
+		"Peer count (light nodes)":   strconv.Itoa(info.peersLight),
+		"Ethstats username":          info.ethstats,
 	}
-	return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s, gastarget=%0.3f MGas, gasprice=%0.3f GWei",
+	if info.peersLight > 0 {
-		info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats, info.gasTarget, info.gasPrice)
+		// Light server enabled
+		report["Listener port (light nodes)"] = strconv.Itoa(info.portLight)
+	}
+	if info.gasTarget > 0 {
+		// Miner or signer node
+		report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
+		report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice)
+
+		if info.etherbase != "" {
+			// Ethash proof-of-work miner
+			report["Ethash directory"] = info.ethashdir
+			report["Miner account"] = info.etherbase
+		}
+		if info.keyJSON != "" {
+			// Clique proof-of-authority signer
+			var key struct {
+				Address string `json:"address"`
+			}
+			if err := json.Unmarshal([]byte(info.keyJSON), &key); err == nil {
+				report["Signer account"] = common.HexToAddress(key.Address).Hex()
+			} else {
+				log.Error("Failed to retrieve signer address", "err", err)
+			}
+		}
+	}
+	return report
 }
 
 // checkNode does a health-check against an boot or seal node server to verify
@@ -223,6 +258,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 	stats := &nodeInfos{
 		genesis:    genesis,
 		datadir:    infos.volumes["/root/.ethereum"],
+		ethashdir:  infos.volumes["/root/.ethash"],
 		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
 		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
 		peersTotal: totalPeers,

cmd/puppeth/module_wallet.go

@@ -0,0 +1,200 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"math/rand"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+)
+
+// walletDockerfile is the Dockerfile required to run a web wallet.
+var walletDockerfile = `
+FROM puppeth/wallet:latest
+
+ADD genesis.json /genesis.json
+
+RUN \
+  echo 'node server.js &'                     > wallet.sh && \
+	echo 'geth --cache 512 init /genesis.json' >> wallet.sh && \
+	echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*"' >> wallet.sh
+
+RUN \
+	sed -i 's/PuppethNetworkID/{{.NetworkID}}/g' dist/js/etherwallet-master.js && \
+	sed -i 's/PuppethNetwork/{{.Network}}/g'     dist/js/etherwallet-master.js && \
+	sed -i 's/PuppethDenom/{{.Denom}}/g'         dist/js/etherwallet-master.js && \
+	sed -i 's/PuppethHost/{{.Host}}/g'           dist/js/etherwallet-master.js && \
+	sed -i 's/PuppethRPCPort/{{.RPCPort}}/g'     dist/js/etherwallet-master.js
+
+ENTRYPOINT ["/bin/sh", "wallet.sh"]
+`
+
+// walletComposefile is the docker-compose.yml file required to deploy and
+// maintain a web wallet.
+var walletComposefile = `
+version: '2'
+services:
+  wallet:
+    build: .
+    image: {{.Network}}/wallet
+    ports:
+      - "{{.NodePort}}:{{.NodePort}}"
+      - "{{.NodePort}}:{{.NodePort}}/udp"
+      - "{{.RPCPort}}:8545"{{if not .VHost}}
+      - "{{.WebPort}}:80"{{end}}
+    volumes:
+      - {{.Datadir}}:/root/.ethereum
+    environment:
+      - NODE_PORT={{.NodePort}}/tcp
+      - STATS={{.Ethstats}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}
+      - VIRTUAL_PORT=80{{end}}
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1m"
+        max-file: "10"
+    restart: always
+`
+
+// deployWallet deploys a new web wallet container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployWallet(client *sshClient, network string, bootnodes []string, config *walletInfos, nocache bool) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(walletDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"Network":   strings.ToTitle(network),
+		"Denom":     strings.ToUpper(network),
+		"NetworkID": config.network,
+		"NodePort":  config.nodePort,
+		"RPCPort":   config.rpcPort,
+		"Bootnodes": strings.Join(bootnodes, ","),
+		"Ethstats":  config.ethstats,
+		"Host":      client.address,
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(walletComposefile)).Execute(composefile, map[string]interface{}{
+		"Datadir":  config.datadir,
+		"Network":  network,
+		"NodePort": config.nodePort,
+		"RPCPort":  config.rpcPort,
+		"VHost":    config.webHost,
+		"WebPort":  config.webPort,
+		"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	files[filepath.Join(workdir, "genesis.json")] = config.genesis
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the boot or seal node service
+	if nocache {
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+	}
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+}
+
+// walletInfos is returned from a web wallet status check to allow reporting
+// various configuration parameters.
+type walletInfos struct {
+	genesis  []byte
+	network  int64
+	datadir  string
+	ethstats string
+	nodePort int
+	rpcPort  int
+	webHost  string
+	webPort  int
+}
+
+// Report converts the typed struct into a plain string->string map, containing
+// most - but not all - fields for reporting to the user.
+func (info *walletInfos) Report() map[string]string {
+	report := map[string]string{
+		"Data directory":         info.datadir,
+		"Ethstats username":      info.ethstats,
+		"Node listener port ":    strconv.Itoa(info.nodePort),
+		"RPC listener port ":     strconv.Itoa(info.rpcPort),
+		"Website address ":       info.webHost,
+		"Website listener port ": strconv.Itoa(info.webPort),
+	}
+	return report
+}
+
+// checkWallet does a health-check against web wallet server to verify whether
+// it's running, and if yes, whether it's responsive.
+func checkWallet(client *sshClient, network string) (*walletInfos, error) {
+	// Inspect a possible web wallet container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_wallet_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	webPort := infos.portmap["80/tcp"]
+	if webPort == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			webPort = proxy.port
+		}
+	}
+	if webPort == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and the config values
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	// Run a sanity check to see if the devp2p and RPC ports are reachable
+	nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
+	if err = checkPort(client.server, nodePort); err != nil {
+		log.Warn(fmt.Sprintf("Wallet devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
+	}
+	rpcPort := infos.portmap["8545/tcp"]
+	if err = checkPort(client.server, rpcPort); err != nil {
+		log.Warn(fmt.Sprintf("Wallet RPC port seems unreachable"), "server", client.server, "port", rpcPort, "err", err)
+	}
+	// Assemble and return the useful infos
+	stats := &walletInfos{
+		datadir:  infos.volumes["/root/.ethereum"],
+		nodePort: nodePort,
+		rpcPort:  rpcPort,
+		webHost:  host,
+		webPort:  webPort,
+		ethstats: infos.envvars["STATS"],
+	}
+	return stats, nil
+}

cmd/puppeth/puppeth.go

@@ -38,7 +38,7 @@ func main() {
 		},
 		cli.IntFlag{
 			Name:  "loglevel",
-			Value: 4,
+			Value: 3,
 			Usage: "log level to emit to the screen",
 		},
 	}

cmd/puppeth/ssh.go

@@ -27,7 +27,6 @@ import (
 	"os/user"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
 	"golang.org/x/crypto/ssh"
@@ -78,14 +77,25 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 	} else {
 		key, err := ssh.ParsePrivateKey(buf)
 		if err != nil {
-			log.Warn("Bad SSH key, falling back to passwords", "path", path, "err", err)
+			fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path)
+			blob, err := terminal.ReadPassword(int(os.Stdin.Fd()))
+			fmt.Println()
+			if err != nil {
+				log.Warn("Couldn't read password", "err", err)
+			}
+			key, err := ssh.ParsePrivateKeyWithPassphrase(buf, blob)
+			if err != nil {
+				log.Warn("Failed to decrypt SSH key, falling back to passwords", "path", path, "err", err)
+			} else {
+				auths = append(auths, ssh.PublicKeys(key))
+			}
 		} else {
 			auths = append(auths, ssh.PublicKeys(key))
 		}
 	}
 	auths = append(auths, ssh.PasswordCallback(func() (string, error) {
 		fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server)
-		blob, err := terminal.ReadPassword(int(syscall.Stdin))
+		blob, err := terminal.ReadPassword(int(os.Stdin.Fd()))
 
 		fmt.Println()
 		return string(blob), err
@@ -106,6 +116,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 	keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
 		// If no public key is known for SSH, ask the user to confirm
 		if pubkey == nil {
+			fmt.Println()
 			fmt.Printf("The authenticity of host '%s (%s)' can't be established.\n", hostname, remote)
 			fmt.Printf("SSH key fingerprint is %s [MD5]\n", ssh.FingerprintLegacyMD5(key))
 			fmt.Printf("Are you sure you want to continue connecting (yes/no)? ")
@@ -205,8 +216,8 @@ func (client *sshClient) Stream(cmd string) error {
 	return session.Run(cmd)
 }
 
-// Upload copied the set of files to a remote server via SCP, creating any non-
+// Upload copies the set of files to a remote server via SCP, creating any non-
-// existing folder in te mean time.
+// existing folders in the mean time.
 func (client *sshClient) Upload(files map[string][]byte) ([]byte, error) {
 	// Establish a single command session
 	session, err := client.client.NewSession()

cmd/puppeth/wizard.go

@@ -28,7 +28,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"syscall"
+	"sync"
 
 	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/core"
@@ -39,12 +39,12 @@ import (
 // config contains all the configurations needed by puppeth that should be saved
 // between sessions.
 type config struct {
-	path      string        // File containing the configuration values
+	path      string   // File containing the configuration values
-	genesis   *core.Genesis // Genesis block to cache for node deploys
+	bootFull  []string // Bootnodes to always connect to by full nodes
-	bootFull  []string      // Bootnodes to always connect to by full nodes
+	bootLight []string // Bootnodes to always connect to by light nodes
-	bootLight []string      // Bootnodes to always connect to by light nodes
+	ethstats  string   // Ethstats settings to cache for node deploys
-	ethstats  string        // Ethstats settings to cache for node deploys
 
+	Genesis *core.Genesis     `json:"genesis,omitempty"` // Genesis block to cache for node deploys
 	Servers map[string][]byte `json:"servers,omitempty"`
 }
 
@@ -76,7 +76,8 @@ type wizard struct {
 	servers  map[string]*sshClient // SSH connections to servers to administer
 	services map[string][]string   // Ethereum services known to be running on servers
 
-	in *bufio.Reader // Wrapper around stdin to allow reading user input
+	in   *bufio.Reader // Wrapper around stdin to allow reading user input
+	lock sync.Mutex    // Lock to protect configs during concurrent service discovery
 }
 
 // read reads a single line from stdin, trimming if from spaces.
@@ -231,7 +232,7 @@ func (w *wizard) readDefaultFloat(def float64) float64 {
 // line and returns it. The input will not be echoed.
 func (w *wizard) readPassword() string {
 	fmt.Printf("> ")
-	text, err := terminal.ReadPassword(int(syscall.Stdin))
+	text, err := terminal.ReadPassword(int(os.Stdin.Fd()))
 	if err != nil {
 		log.Crit("Failed to read password", "err", err)
 	}
@@ -302,8 +303,10 @@ func (w *wizard) readJSON() string {
 }
 
 // readIPAddress reads a single line from stdin, trimming if from spaces and
-// converts it to a network IP address.
+// returning it if it's convertible to an IP address. The reason for keeping
-func (w *wizard) readIPAddress() net.IP {
+// the user input format instead of returning a Go net.IP is to match with
+// weird formats used by ethstats, which compares IPs textually, not by value.
+func (w *wizard) readIPAddress() string {
 	for {
 		// Read the IP address from the user
 		fmt.Printf("> ")
@@ -312,14 +315,13 @@ func (w *wizard) readIPAddress() net.IP {
 			log.Crit("Failed to read user input", "err", err)
 		}
 		if text = strings.TrimSpace(text); text == "" {
-			return nil
+			return ""
 		}
 		// Make sure it looks ok and return it if so
-		ip := net.ParseIP(text)
+		if ip := net.ParseIP(text); ip == nil {
-		if ip == nil {
 			log.Error("Invalid IP address, please retry")
 			continue
 		}
-		return ip
+		return text
 	}
 }

cmd/puppeth/wizard_dashboard.go

@@ -40,6 +40,8 @@ func (w *wizard) deployDashboard() {
 			host: client.server,
 		}
 	}
+	existed := err == nil
+
 	// Figure out which port to listen on
 	fmt.Println()
 	fmt.Printf("Which port should the dashboard listen on? (default = %d)\n", infos.port)
@@ -58,7 +60,6 @@ func (w *wizard) deployDashboard() {
 			available[service] = append(available[service], server)
 		}
 	}
-	listing := make(map[string]string)
 	for _, service := range []string{"ethstats", "explorer", "wallet", "faucet"} {
 		// Gather all the locally hosted pages of this type
 		var pages []string
@@ -74,6 +75,14 @@ func (w *wizard) deployDashboard() {
 				if infos, err := checkEthstats(client, w.network); err == nil {
 					port = infos.port
 				}
+			case "explorer":
+				if infos, err := checkExplorer(client, w.network); err == nil {
+					port = infos.webPort
+				}
+			case "wallet":
+				if infos, err := checkWallet(client, w.network); err == nil {
+					port = infos.webPort
+				}
 			case "faucet":
 				if infos, err := checkFaucet(client, w.network); err == nil {
 					port = infos.port
@@ -101,26 +110,43 @@ func (w *wizard) deployDashboard() {
 			log.Error("Invalid listing choice, aborting")
 			return
 		}
+		var page string
 		switch {
 		case choice <= len(pages):
-			listing[service] = pages[choice-1]
+			page = pages[choice-1]
 		case choice == len(pages)+1:
 			fmt.Println()
 			fmt.Printf("Which address is the external %s service at?\n", service)
-			listing[service] = w.readString()
+			page = w.readString()
 		default:
 			// No service hosting for this
 		}
+		// Save the users choice
+		switch service {
+		case "ethstats":
+			infos.ethstats = page
+		case "explorer":
+			infos.explorer = page
+		case "wallet":
+			infos.wallet = page
+		case "faucet":
+			infos.faucet = page
+		}
 	}
 	// If we have ethstats running, ask whether to make the secret public or not
-	var ethstats bool
 	if w.conf.ethstats != "" {
 		fmt.Println()
 		fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)")
-		ethstats = w.readDefaultString("y") == "y"
+		infos.trusted = w.readDefaultString("y") == "y"
 	}
 	// Try to deploy the dashboard container on the host
-	if out, err := deployDashboard(client, w.network, infos.port, infos.host, listing, &w.conf, ethstats); err != nil {
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the dashboard be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
+	if out, err := deployDashboard(client, w.network, &w.conf, infos, nocache); err != nil {
 		log.Error("Failed to deploy dashboard container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
@@ -128,5 +154,5 @@ func (w *wizard) deployDashboard() {
 		return
 	}
 	// All ok, run a network scan to pick any changes up
-	w.networkStats(false)
+	w.networkStats()
 }

cmd/puppeth/wizard_ethstats.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
 )
@@ -41,6 +42,8 @@ func (w *wizard) deployEthstats() {
 			secret: "",
 		}
 	}
+	existed := err == nil
+
 	// Figure out which port to listen on
 	fmt.Println()
 	fmt.Printf("Which port should ethstats listen on? (default = %d)\n", infos.port)
@@ -61,29 +64,57 @@ func (w *wizard) deployEthstats() {
 		infos.secret = w.readDefaultString(infos.secret)
 	}
 	// Gather any blacklists to ban from reporting
-	fmt.Println()
+	if existed {
-	fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned)
-	if w.readDefaultString("y") != "y" {
-		infos.banned = nil
-
 		fmt.Println()
-		fmt.Println("Which IP addresses should be blacklisted?")
+		fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned)
-		for {
+		if w.readDefaultString("y") != "y" {
-			if ip := w.readIPAddress(); ip != nil {
+			// The user might want to clear the entire list, although generally probably not
-				infos.banned = append(infos.banned, ip.String())
+			fmt.Println()
-				continue
+			fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n")
+			if w.readDefaultString("n") != "n" {
+				infos.banned = nil
 			}
-			break
+			// Offer the user to explicitly add/remove certain IP addresses
+			fmt.Println()
+			fmt.Println("Which additional IP addresses should be blacklisted?")
+			for {
+				if ip := w.readIPAddress(); ip != "" {
+					infos.banned = append(infos.banned, ip)
+					continue
+				}
+				break
+			}
+			fmt.Println()
+			fmt.Println("Which IP addresses should not be blacklisted?")
+			for {
+				if ip := w.readIPAddress(); ip != "" {
+					for i, addr := range infos.banned {
+						if ip == addr {
+							infos.banned = append(infos.banned[:i], infos.banned[i+1:]...)
+							break
+						}
+					}
+					continue
+				}
+				break
+			}
+			sort.Strings(infos.banned)
 		}
 	}
 	// Try to deploy the ethstats server on the host
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the ethstats be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
 	trusted := make([]string, 0, len(w.servers))
 	for _, client := range w.servers {
 		if client != nil {
 			trusted = append(trusted, client.address)
 		}
 	}
-	if out, err := deployEthstats(client, w.network, infos.port, infos.secret, infos.host, trusted, infos.banned); err != nil {
+	if out, err := deployEthstats(client, w.network, infos.port, infos.secret, infos.host, trusted, infos.banned, nocache); err != nil {
 		log.Error("Failed to deploy ethstats container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
@@ -91,5 +122,5 @@ func (w *wizard) deployEthstats() {
 		return
 	}
 	// All ok, run a network scan to pick any changes up
-	w.networkStats(false)
+	w.networkStats()
 }

cmd/puppeth/wizard_explorer.go

@@ -0,0 +1,117 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+)
+
+// deployExplorer creates a new block explorer based on some user input.
+func (w *wizard) deployExplorer() {
+	// Do some sanity check before the user wastes time on input
+	if w.conf.Genesis == nil {
+		log.Error("No genesis block configured")
+		return
+	}
+	if w.conf.ethstats == "" {
+		log.Error("No ethstats server configured")
+		return
+	}
+	if w.conf.Genesis.Config.Ethash == nil {
+		log.Error("Only ethash network supported")
+		return
+	}
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active node configurations from the server
+	infos, err := checkExplorer(client, w.network)
+	if err != nil {
+		infos = &explorerInfos{
+			nodePort: 30303, webPort: 80, webHost: client.server,
+		}
+	}
+	existed := err == nil
+
+	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootFull)
+	if err != nil {
+		log.Error("Failed to create chain spec for explorer", "err", err)
+		return
+	}
+	chain, _ := json.MarshalIndent(chainspec, "", "  ")
+
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.webPort)
+	infos.webPort = w.readDefaultInt(infos.webPort)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.webHost, err = w.ensureVirtualHost(client, infos.webPort, infos.webHost); err != nil {
+		log.Error("Failed to decide on explorer host", "err", err)
+		return
+	}
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		infos.datadir = w.readDefaultString(infos.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.nodePort)
+	infos.nodePort = w.readDefaultInt(infos.nodePort)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.ethstats == "" {
+		fmt.Printf("What should the explorer be called on the stats page?\n")
+		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.ethstats)
+		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+	}
+	// Try to deploy the explorer on the host
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
+	if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil {
+		log.Error("Failed to deploy explorer container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	log.Info("Waiting for node to finish booting")
+	time.Sleep(3 * time.Second)
+
+	w.networkStats()
+}

cmd/puppeth/wizard_faucet.go

@@ -19,7 +19,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"net/http"
 
 	"github.com/EthereumCommonwealth/go-callisto/accounts/keystore"
 	"github.com/EthereumCommonwealth/go-callisto/log"
@@ -47,8 +46,10 @@ func (w *wizard) deployFaucet() {
 			tiers:   3,
 		}
 	}
-	infos.node.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	existed := err == nil
-	infos.node.network = w.conf.genesis.Config.ChainId.Int64()
+
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", "  ")
+	infos.node.network = w.conf.Genesis.Config.ChainId.Int64()
 
 	// Figure out which port to listen on
 	fmt.Println()
@@ -60,7 +61,7 @@ func (w *wizard) deployFaucet() {
 		log.Error("Failed to decide on faucet host", "err", err)
 		return
 	}
-	// Port and proxy settings retrieved, figure out the funcing amount per perdion configurations
+	// Port and proxy settings retrieved, figure out the funding amount per period configurations
 	fmt.Println()
 	fmt.Printf("How many Ethers to release per request? (default = %d)\n", infos.amount)
 	infos.amount = w.readDefaultInt(infos.amount)
@@ -76,47 +77,6 @@ func (w *wizard) deployFaucet() {
 		log.Error("At least one funding tier must be set")
 		return
 	}
-	// Accessing GitHub gists requires API authorization, retrieve it
-	if infos.githubUser != "" {
-		fmt.Println()
-		fmt.Printf("Reuse previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser)
-		if w.readDefaultString("y") != "y" {
-			infos.githubUser, infos.githubToken = "", ""
-		}
-	}
-	if infos.githubUser == "" {
-		// No previous authorization (or new one requested)
-		fmt.Println()
-		fmt.Println("Which GitHub user to verify Gists through?")
-		infos.githubUser = w.readString()
-
-		fmt.Println()
-		fmt.Println("What is the GitHub personal access token of the user? (won't be echoed)")
-		infos.githubToken = w.readPassword()
-
-		// Do a sanity check query against github to ensure it's valid
-		req, _ := http.NewRequest("GET", "https://api.github.com/user", nil)
-		req.SetBasicAuth(infos.githubUser, infos.githubToken)
-		res, err := http.DefaultClient.Do(req)
-		if err != nil {
-			log.Error("Failed to verify GitHub authentication", "err", err)
-			return
-		}
-		defer res.Body.Close()
-
-		var msg struct {
-			Login   string `json:"login"`
-			Message string `json:"message"`
-		}
-		if err = json.NewDecoder(res.Body).Decode(&msg); err != nil {
-			log.Error("Failed to decode authorization response", "err", err)
-			return
-		}
-		if msg.Login != infos.githubUser {
-			log.Error("GitHub authorization failed", "user", infos.githubUser, "message", msg.Message)
-			return
-		}
-	}
 	// Accessing the reCaptcha service requires API authorizations, request it
 	if infos.captchaToken != "" {
 		fmt.Println()
@@ -129,7 +89,9 @@ func (w *wizard) deployFaucet() {
 		// No previous authorization (or old one discarded)
 		fmt.Println()
 		fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)")
-		if w.readDefaultString("n") == "y" {
+		if w.readDefaultString("n") == "n" {
+			log.Warn("Users will be able to requests funds via automated scripts")
+		} else {
 			// Captcha protection explicitly requested, read the site and secret keys
 			fmt.Println()
 			fmt.Printf("What is the reCaptcha site key to authenticate human users?\n")
@@ -175,7 +137,7 @@ func (w *wizard) deployFaucet() {
 			}
 		}
 	}
-	if infos.node.keyJSON == "" {
+	for i := 0; i < 3 && infos.node.keyJSON == ""; i++ {
 		fmt.Println()
 		fmt.Println("Please paste the faucet's funding account key JSON:")
 		infos.node.keyJSON = w.readJSON()
@@ -186,11 +148,27 @@ func (w *wizard) deployFaucet() {
 
 		if _, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil {
 			log.Error("Failed to decrypt key with given passphrase")
-			return
+			infos.node.keyJSON = ""
+			infos.node.keyPass = ""
 		}
 	}
+	// Check if the user wants to run the faucet in debug mode (noauth)
+	noauth := "n"
+	if infos.noauth {
+		noauth = "y"
+	}
+	fmt.Println()
+	fmt.Printf("Permit non-authenticated funding requests (y/n)? (default = %v)\n", infos.noauth)
+	infos.noauth = w.readDefaultString(noauth) != "n"
+
 	// Try to deploy the faucet server on the host
-	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos); err != nil {
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
+	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos, nocache); err != nil {
 		log.Error("Failed to deploy faucet container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
@@ -198,5 +176,5 @@ func (w *wizard) deployFaucet() {
 		return
 	}
 	// All ok, run a network scan to pick any changes up
-	w.networkStats(false)
+	w.networkStats()
 }

cmd/puppeth/wizard_genesis.go

@@ -37,7 +37,7 @@ func (w *wizard) makeGenesis() {
 	genesis := &core.Genesis{
 		Timestamp:  uint64(time.Now().Unix()),
 		GasLimit:   4700000,
-		Difficulty: big.NewInt(1048576),
+		Difficulty: big.NewInt(524288),
 		Alloc:      make(core.GenesisAlloc),
 		Config: &params.ChainConfig{
 			HomesteadBlock: big.NewInt(1),
@@ -118,24 +118,16 @@ func (w *wizard) makeGenesis() {
 	for i := int64(0); i < 256; i++ {
 		genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
 	}
-	fmt.Println()
-
 	// Query the user for some custom extras
 	fmt.Println()
 	fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)")
 	genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536))))
 
-	fmt.Println()
-	fmt.Println("Anything fun to embed into the genesis block? (max 32 bytes)")
-
-	extra := w.read()
-	if len(extra) > 32 {
-		extra = extra[:32]
-	}
-	genesis.ExtraData = append([]byte(extra), genesis.ExtraData[len(extra):]...)
-
 	// All done, store the genesis and flush to disk
-	w.conf.genesis = genesis
+	log.Info("Configured new genesis block")
+
+	w.conf.Genesis = genesis
+	w.conf.flush()
 }
 
 // manageGenesis permits the modification of chain configuration parameters in
@@ -145,44 +137,56 @@ func (w *wizard) manageGenesis() {
 	fmt.Println()
 	fmt.Println(" 1. Modify existing fork rules")
 	fmt.Println(" 2. Export genesis configuration")
+	fmt.Println(" 3. Remove genesis configuration")
 
 	choice := w.read()
 	switch {
 	case choice == "1":
 		// Fork rule updating requested, iterate over each fork
 		fmt.Println()
-		fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.genesis.Config.HomesteadBlock)
+		fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.Genesis.Config.HomesteadBlock)
-		w.conf.genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.genesis.Config.HomesteadBlock)
+		w.conf.Genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.Genesis.Config.HomesteadBlock)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP150Block)
+		fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block)
-		w.conf.genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP150Block)
+		w.conf.Genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP150Block)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP155Block)
+		fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block)
-		w.conf.genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP155Block)
+		w.conf.Genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP155Block)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.genesis.Config.EIP158Block)
+		fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block)
-		w.conf.genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.genesis.Config.EIP158Block)
+		w.conf.Genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP158Block)
 
 		fmt.Println()
-		fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.genesis.Config.ByzantiumBlock)
+		fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock)
-		w.conf.genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.genesis.Config.ByzantiumBlock)
+		w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock)
 
-		out, _ := json.MarshalIndent(w.conf.genesis.Config, "", "  ")
+		out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", "  ")
 		fmt.Printf("Chain configuration updated:\n\n%s\n", out)
 
 	case choice == "2":
 		// Save whatever genesis configuration we currently have
 		fmt.Println()
 		fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
-		out, _ := json.MarshalIndent(w.conf.genesis, "", "  ")
+		out, _ := json.MarshalIndent(w.conf.Genesis, "", "  ")
 		if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
 			log.Error("Failed to save genesis file", "err", err)
 		}
 		log.Info("Exported existing genesis block")
 
+	case choice == "3":
+		// Make sure we don't have any services running
+		if len(w.conf.servers()) > 0 {
+			log.Error("Genesis reset requires all services and servers torn down")
+			return
+		}
+		log.Info("Genesis block destroyed")
+
+		w.conf.Genesis = nil
+		w.conf.flush()
+
 	default:
 		log.Error("That's not something I can do")
 	}

cmd/puppeth/wizard_intro.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
 )
@@ -63,7 +64,7 @@ func (w *wizard) run() {
 		for {
 			w.network = w.readString()
 			if !strings.Contains(w.network, " ") {
-				fmt.Printf("Sweet, you can set this via --network=%s next time!\n\n", w.network)
+				fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network)
 				break
 			}
 			log.Error("I also like to live dangerously, still no spaces")
@@ -80,22 +81,33 @@ func (w *wizard) run() {
 	} else if err := json.Unmarshal(blob, &w.conf); err != nil {
 		log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err)
 	} else {
+		// Dial all previously known servers concurrently
+		var pend sync.WaitGroup
 		for server, pubkey := range w.conf.Servers {
-			log.Info("Dialing previously configured server", "server", server)
+			pend.Add(1)
-			client, err := dial(server, pubkey)
+
-			if err != nil {
+			go func(server string, pubkey []byte) {
-				log.Error("Previous server unreachable", "server", server, "err", err)
+				defer pend.Done()
-			}
+
-			w.servers[server] = client
+				log.Info("Dialing previously configured server", "server", server)
+				client, err := dial(server, pubkey)
+				if err != nil {
+					log.Error("Previous server unreachable", "server", server, "err", err)
+				}
+				w.lock.Lock()
+				w.servers[server] = client
+				w.lock.Unlock()
+			}(server, pubkey)
 		}
-		w.networkStats(false)
+		pend.Wait()
+		w.networkStats()
 	}
 	// Basics done, loop ad infinitum about what to do
 	for {
 		fmt.Println()
 		fmt.Println("What would you like to do? (default = stats)")
 		fmt.Println(" 1. Show network stats")
-		if w.conf.genesis == nil {
+		if w.conf.Genesis == nil {
 			fmt.Println(" 2. Configure new genesis")
 		} else {
 			fmt.Println(" 2. Manage existing genesis")
@@ -110,15 +122,14 @@ func (w *wizard) run() {
 		} else {
 			fmt.Println(" 4. Manage network components")
 		}
-		//fmt.Println(" 5. ProTips for common usecases")
 
 		choice := w.read()
 		switch {
 		case choice == "" || choice == "1":
-			w.networkStats(false)
+			w.networkStats()
 
 		case choice == "2":
-			if w.conf.genesis == nil {
+			if w.conf.Genesis == nil {
 				w.makeGenesis()
 			} else {
 				w.manageGenesis()
@@ -126,7 +137,7 @@ func (w *wizard) run() {
 		case choice == "3":
 			if len(w.servers) == 0 {
 				if w.makeServer() != "" {
-					w.networkStats(false)
+					w.networkStats()
 				}
 			} else {
 				w.manageServers()
@@ -138,9 +149,6 @@ func (w *wizard) run() {
 				w.manageComponents()
 			}
 
-		case choice == "5":
-			w.networkStats(true)
-
 		default:
 			log.Error("That's not something I can do")
 		}

cmd/puppeth/wizard_netstats.go

@@ -18,9 +18,10 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
 	"os"
+	"sort"
 	"strings"
+	"sync"
 
 	"github.com/EthereumCommonwealth/go-callisto/core"
 	"github.com/EthereumCommonwealth/go-callisto/log"
@@ -29,127 +30,257 @@ import (
 
 // networkStats verifies the status of network components and generates a protip
 // configuration set to give users hints on how to do various tasks.
-func (w *wizard) networkStats(tips bool) {
+func (w *wizard) networkStats() {
 	if len(w.servers) == 0 {
-		log.Error("No remote machines to gather stats from")
+		log.Info("No remote machines to gather stats from")
 		return
 	}
-	protips := new(protips)
+	// Clear out some previous configs to refill from current scan
+	w.conf.ethstats = ""
+	w.conf.bootFull = w.conf.bootFull[:0]
+	w.conf.bootLight = w.conf.bootLight[:0]
 
 	// Iterate over all the specified hosts and check their status
-	stats := tablewriter.NewWriter(os.Stdout)
+	var pend sync.WaitGroup
-	stats.SetHeader([]string{"Server", "IP", "Status", "Service", "Details"})
-	stats.SetColWidth(100)
 
+	stats := make(serverStats)
 	for server, pubkey := range w.conf.Servers {
-		client := w.servers[server]
+		pend.Add(1)
-		logger := log.New("server", server)
-		logger.Info("Starting remote server health-check")
 
-		// If the server is not connected, try to connect again
+		// Gather the service stats for each server concurrently
-		if client == nil {
+		go func(server string, pubkey []byte) {
-			conn, err := dial(server, pubkey)
+			defer pend.Done()
-			if err != nil {
-				logger.Error("Failed to establish remote connection", "err", err)
-				stats.Append([]string{server, "", err.Error(), "", ""})
-				continue
-			}
-			client = conn
-		}
-		// Client connected one way or another, run health-checks
-		services := make(map[string]string)
-		logger.Debug("Checking for nginx availability")
-		if infos, err := checkNginx(client, w.network); err != nil {
-			if err != ErrServiceUnknown {
-				services["nginx"] = err.Error()
-			}
-		} else {
-			services["nginx"] = infos.String()
-		}
-		logger.Debug("Checking for ethstats availability")
-		if infos, err := checkEthstats(client, w.network); err != nil {
-			if err != ErrServiceUnknown {
-				services["ethstats"] = err.Error()
-			}
-		} else {
-			services["ethstats"] = infos.String()
-			protips.ethstats = infos.config
-		}
-		logger.Debug("Checking for bootnode availability")
-		if infos, err := checkNode(client, w.network, true); err != nil {
-			if err != ErrServiceUnknown {
-				services["bootnode"] = err.Error()
-			}
-		} else {
-			services["bootnode"] = infos.String()
 
-			protips.genesis = string(infos.genesis)
+			stat := w.gatherStats(server, pubkey, w.servers[server])
-			protips.bootFull = append(protips.bootFull, infos.enodeFull)
+
-			if infos.enodeLight != "" {
+			// All status checks complete, report and check next server
-				protips.bootLight = append(protips.bootLight, infos.enodeLight)
+			w.lock.Lock()
+			defer w.lock.Unlock()
+
+			delete(w.services, server)
+			for service := range stat.services {
+				w.services[server] = append(w.services[server], service)
 			}
-		}
+			stats[server] = stat
-		logger.Debug("Checking for sealnode availability")
+		}(server, pubkey)
-		if infos, err := checkNode(client, w.network, false); err != nil {
-			if err != ErrServiceUnknown {
-				services["sealnode"] = err.Error()
-			}
-		} else {
-			services["sealnode"] = infos.String()
-			protips.genesis = string(infos.genesis)
-		}
-		logger.Debug("Checking for faucet availability")
-		if infos, err := checkFaucet(client, w.network); err != nil {
-			if err != ErrServiceUnknown {
-				services["faucet"] = err.Error()
-			}
-		} else {
-			services["faucet"] = infos.String()
-		}
-		logger.Debug("Checking for dashboard availability")
-		if infos, err := checkDashboard(client, w.network); err != nil {
-			if err != ErrServiceUnknown {
-				services["dashboard"] = err.Error()
-			}
-		} else {
-			services["dashboard"] = infos.String()
-		}
-		// All status checks complete, report and check next server
-		delete(w.services, server)
-		for service := range services {
-			w.services[server] = append(w.services[server], service)
-		}
-		server, address := client.server, client.address
-		for service, status := range services {
-			stats.Append([]string{server, address, "online", service, status})
-			server, address = "", ""
-		}
-		if len(services) == 0 {
-			stats.Append([]string{server, address, "online", "", ""})
-		}
 	}
-	// If a genesis block was found, load it into our configs
+	pend.Wait()
-	if protips.genesis != "" && w.conf.genesis == nil {
-		genesis := new(core.Genesis)
-		if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
-			log.Error("Failed to parse remote genesis", "err", err)
-		} else {
-			w.conf.genesis = genesis
-			protips.network = genesis.Config.ChainId.Int64()
-		}
-	}
-	if protips.ethstats != "" {
-		w.conf.ethstats = protips.ethstats
-	}
-	w.conf.bootFull = protips.bootFull
-	w.conf.bootLight = protips.bootLight
 
 	// Print any collected stats and return
-	if !tips {
+	stats.render()
-		stats.Render()
+}
-	} else {
+
-		protips.print(w.network)
+// gatherStats gathers service statistics for a particular remote server.
+func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *serverStat {
+	// Gather some global stats to feed into the wizard
+	var (
+		genesis   string
+		ethstats  string
+		bootFull  []string
+		bootLight []string
+	)
+	// Ensure a valid SSH connection to the remote server
+	logger := log.New("server", server)
+	logger.Info("Starting remote server health-check")
+
+	stat := &serverStat{
+		address:  client.address,
+		services: make(map[string]map[string]string),
 	}
+	if client == nil {
+		conn, err := dial(server, pubkey)
+		if err != nil {
+			logger.Error("Failed to establish remote connection", "err", err)
+			stat.failure = err.Error()
+			return stat
+		}
+		client = conn
+	}
+	// Client connected one way or another, run health-checks
+	logger.Debug("Checking for nginx availability")
+	if infos, err := checkNginx(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["nginx"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["nginx"] = infos.Report()
+	}
+	logger.Debug("Checking for ethstats availability")
+	if infos, err := checkEthstats(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["ethstats"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["ethstats"] = infos.Report()
+		ethstats = infos.config
+	}
+	logger.Debug("Checking for bootnode availability")
+	if infos, err := checkNode(client, w.network, true); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["bootnode"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["bootnode"] = infos.Report()
+
+		genesis = string(infos.genesis)
+		bootFull = append(bootFull, infos.enodeFull)
+		if infos.enodeLight != "" {
+			bootLight = append(bootLight, infos.enodeLight)
+		}
+	}
+	logger.Debug("Checking for sealnode availability")
+	if infos, err := checkNode(client, w.network, false); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["sealnode"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["sealnode"] = infos.Report()
+		genesis = string(infos.genesis)
+	}
+	logger.Debug("Checking for explorer availability")
+	if infos, err := checkExplorer(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["explorer"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["explorer"] = infos.Report()
+	}
+	logger.Debug("Checking for wallet availability")
+	if infos, err := checkWallet(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["wallet"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["wallet"] = infos.Report()
+	}
+	logger.Debug("Checking for faucet availability")
+	if infos, err := checkFaucet(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["faucet"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["faucet"] = infos.Report()
+	}
+	logger.Debug("Checking for dashboard availability")
+	if infos, err := checkDashboard(client, w.network); err != nil {
+		if err != ErrServiceUnknown {
+			stat.services["dashboard"] = map[string]string{"offline": err.Error()}
+		}
+	} else {
+		stat.services["dashboard"] = infos.Report()
+	}
+	// Feed and newly discovered information into the wizard
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if genesis != "" && w.conf.Genesis == nil {
+		g := new(core.Genesis)
+		if err := json.Unmarshal([]byte(genesis), g); err != nil {
+			log.Error("Failed to parse remote genesis", "err", err)
+		} else {
+			w.conf.Genesis = g
+		}
+	}
+	if ethstats != "" {
+		w.conf.ethstats = ethstats
+	}
+	w.conf.bootFull = append(w.conf.bootFull, bootFull...)
+	w.conf.bootLight = append(w.conf.bootLight, bootLight...)
+
+	return stat
+}
+
+// serverStat is a collection of service configuration parameters and health
+// check reports to print to the user.
+type serverStat struct {
+	address  string
+	failure  string
+	services map[string]map[string]string
+}
+
+// serverStats is a collection of server stats for multiple hosts.
+type serverStats map[string]*serverStat
+
+// render converts the gathered statistics into a user friendly tabular report
+// and prints it to the standard output.
+func (stats serverStats) render() {
+	// Start gathering service statistics and config parameters
+	table := tablewriter.NewWriter(os.Stdout)
+
+	table.SetHeader([]string{"Server", "Address", "Service", "Config", "Value"})
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	table.SetColWidth(100)
+
+	// Find the longest lines for all columns for the hacked separator
+	separator := make([]string, 5)
+	for server, stat := range stats {
+		if len(server) > len(separator[0]) {
+			separator[0] = strings.Repeat("-", len(server))
+		}
+		if len(stat.address) > len(separator[1]) {
+			separator[1] = strings.Repeat("-", len(stat.address))
+		}
+		for service, configs := range stat.services {
+			if len(service) > len(separator[2]) {
+				separator[2] = strings.Repeat("-", len(service))
+			}
+			for config, value := range configs {
+				if len(config) > len(separator[3]) {
+					separator[3] = strings.Repeat("-", len(config))
+				}
+				if len(value) > len(separator[4]) {
+					separator[4] = strings.Repeat("-", len(value))
+				}
+			}
+		}
+	}
+	// Fill up the server report in alphabetical order
+	servers := make([]string, 0, len(stats))
+	for server := range stats {
+		servers = append(servers, server)
+	}
+	sort.Strings(servers)
+
+	for i, server := range servers {
+		// Add a separator between all servers
+		if i > 0 {
+			table.Append(separator)
+		}
+		// Fill up the service report in alphabetical order
+		services := make([]string, 0, len(stats[server].services))
+		for service := range stats[server].services {
+			services = append(services, service)
+		}
+		sort.Strings(services)
+
+		if len(services) == 0 {
+			table.Append([]string{server, stats[server].address, "", "", ""})
+		}
+		for j, service := range services {
+			// Add an empty line between all services
+			if j > 0 {
+				table.Append([]string{"", "", "", separator[3], separator[4]})
+			}
+			// Fill up the config report in alphabetical order
+			configs := make([]string, 0, len(stats[server].services[service]))
+			for service := range stats[server].services[service] {
+				configs = append(configs, service)
+			}
+			sort.Strings(configs)
+
+			for k, config := range configs {
+				switch {
+				case j == 0 && k == 0:
+					table.Append([]string{server, stats[server].address, service, config, stats[server].services[service][config]})
+				case k == 0:
+					table.Append([]string{"", "", service, config, stats[server].services[service][config]})
+				default:
+					table.Append([]string{"", "", "", config, stats[server].services[service][config]})
+				}
+			}
+		}
+	}
+	table.Render()
 }
 
 // protips contains a collection of network infos to report pro-tips
@@ -161,75 +292,3 @@ type protips struct {
 	bootLight []string
 	ethstats  string
 }
-
-// print analyzes the network information available and prints a collection of
-// pro tips for the user's consideration.
-func (p *protips) print(network string) {
-	// If a known genesis block is available, display it and prepend an init command
-	fullinit, lightinit := "", ""
-	if p.genesis != "" {
-		fullinit = fmt.Sprintf("geth --datadir=$HOME/.%s init %s.json && ", network, network)
-		lightinit = fmt.Sprintf("geth --datadir=$HOME/.%s --light init %s.json && ", network, network)
-	}
-	// If an ethstats server is available, add the ethstats flag
-	statsflag := ""
-	if p.ethstats != "" {
-		if strings.Contains(p.ethstats, " ") {
-			statsflag = fmt.Sprintf(` --ethstats="yournode:%s"`, p.ethstats)
-		} else {
-			statsflag = fmt.Sprintf(` --ethstats=yournode:%s`, p.ethstats)
-		}
-	}
-	// If bootnodes have been specified, add the bootnode flag
-	bootflagFull := ""
-	if len(p.bootFull) > 0 {
-		bootflagFull = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootFull, ","))
-	}
-	bootflagLight := ""
-	if len(p.bootLight) > 0 {
-		bootflagLight = fmt.Sprintf(` --bootnodes %s`, strings.Join(p.bootLight, ","))
-	}
-	// Assemble all the known pro-tips
-	var tasks, tips []string
-
-	tasks = append(tasks, "Run an archive node with historical data")
-	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=1024%s%s", fullinit, p.network, network, statsflag, bootflagFull))
-
-	tasks = append(tasks, "Run a full node with recent data only")
-	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=512 --fast%s%s", fullinit, p.network, network, statsflag, bootflagFull))
-
-	tasks = append(tasks, "Run a light node with on demand retrievals")
-	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
-
-	tasks = append(tasks, "Run an embedded node with constrained memory")
-	tips = append(tips, fmt.Sprintf("%sgeth --networkid=%d --datadir=$HOME/.%s --cache=32 --light%s%s", lightinit, p.network, network, statsflag, bootflagLight))
-
-	// If the tips are short, display in a table
-	short := true
-	for _, tip := range tips {
-		if len(tip) > 100 {
-			short = false
-			break
-		}
-	}
-	fmt.Println()
-	if short {
-		howto := tablewriter.NewWriter(os.Stdout)
-		howto.SetHeader([]string{"Fun tasks for you", "Tips on how to"})
-		howto.SetColWidth(100)
-
-		for i := 0; i < len(tasks); i++ {
-			howto.Append([]string{tasks[i], tips[i]})
-		}
-		howto.Render()
-		return
-	}
-	// Meh, tips got ugly, split into many lines
-	for i := 0; i < len(tasks); i++ {
-		fmt.Println(tasks[i])
-		fmt.Println(strings.Repeat("-", len(tasks[i])))
-		fmt.Println(tips[i])
-		fmt.Println()
-		fmt.Println()
-	}
-}

cmd/puppeth/wizard_network.go

@@ -53,12 +53,12 @@ func (w *wizard) manageServers() {
 		w.conf.flush()
 
 		log.Info("Disconnected existing server", "server", server)
-		w.networkStats(false)
+		w.networkStats()
 		return
 	}
 	// If the user requested connecting a new server, do it
 	if w.makeServer() != "" {
-		w.networkStats(false)
+		w.networkStats()
 	}
 }
 
@@ -71,7 +71,7 @@ func (w *wizard) makeServer() string {
 	fmt.Println()
 	fmt.Println("Please enter remote server's address:")
 
-	// Read and fial the server to ensure docker is present
+	// Read and dial the server to ensure docker is present
 	input := w.readString()
 
 	client, err := dial(input, nil)
@@ -174,9 +174,10 @@ func (w *wizard) deployComponent() {
 	fmt.Println(" 1. Ethstats  - Network monitoring tool")
 	fmt.Println(" 2. Bootnode  - Entry point of the network")
 	fmt.Println(" 3. Sealer    - Full node minting new blocks")
-	fmt.Println(" 4. Wallet    - Browser wallet for quick sends (todo)")
+	fmt.Println(" 4. Explorer  - Chain analysis webservice (ethash only)")
-	fmt.Println(" 5. Faucet    - Crypto faucet to give away funds")
+	fmt.Println(" 5. Wallet    - Browser wallet for quick sends")
-	fmt.Println(" 6. Dashboard - Website listing above web-services")
+	fmt.Println(" 6. Faucet    - Crypto faucet to give away funds")
+	fmt.Println(" 7. Dashboard - Website listing above web-services")
 
 	switch w.read() {
 	case "1":
@@ -186,9 +187,12 @@ func (w *wizard) deployComponent() {
 	case "3":
 		w.deployNode(false)
 	case "4":
+		w.deployExplorer()
 	case "5":
-		w.deployFaucet()
+		w.deployWallet()
 	case "6":
+		w.deployFaucet()
+	case "7":
 		w.deployDashboard()
 	default:
 		log.Error("That's not something I can do")

cmd/puppeth/wizard_nginx.go

@@ -29,7 +29,8 @@ import (
 //
 // If the user elects not to use a reverse proxy, an empty hostname is returned!
 func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (string, error) {
-	if proxy, _ := checkNginx(client, w.network); proxy != nil {
+	proxy, _ := checkNginx(client, w.network)
+	if proxy != nil {
 		// Reverse proxy is running, if ports match, we need a virtual host
 		if proxy.port == port {
 			fmt.Println()
@@ -41,7 +42,13 @@ func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (str
 	fmt.Println()
 	fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)")
 	if w.readDefaultString("y") == "y" {
-		if out, err := deployNginx(client, w.network, port); err != nil {
+		nocache := false
+		if proxy != nil {
+			fmt.Println()
+			fmt.Printf("Should the reverse-proxy be rebuilt from scratch (y/n)? (default = no)\n")
+			nocache = w.readDefaultString("n") != "n"
+		}
+		if out, err := deployNginx(client, w.network, port, nocache); err != nil {
 			log.Error("Failed to deploy reverse-proxy", "err", err)
 			if len(out) > 0 {
 				fmt.Printf("%s\n", out)

cmd/puppeth/wizard_node.go

@@ -29,7 +29,7 @@ import (
 // deployNode creates a new node configuration based on some user input.
 func (w *wizard) deployNode(boot bool) {
 	// Do some sanity check before the user wastes time on input
-	if w.conf.genesis == nil {
+	if w.conf.Genesis == nil {
 		log.Error("No genesis block configured")
 		return
 	}
@@ -44,7 +44,7 @@ func (w *wizard) deployNode(boot bool) {
 	}
 	client := w.servers[server]
 
-	// Retrieve any active ethstats configurations from the server
+	// Retrieve any active node configurations from the server
 	infos, err := checkNode(client, w.network, boot)
 	if err != nil {
 		if boot {
@@ -53,8 +53,10 @@ func (w *wizard) deployNode(boot bool) {
 			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
 		}
 	}
-	infos.genesis, _ = json.MarshalIndent(w.conf.genesis, "", "  ")
+	existed := err == nil
-	infos.network = w.conf.genesis.Config.ChainId.Int64()
+
+	infos.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", "  ")
+	infos.network = w.conf.Genesis.Config.ChainId.Int64()
 
 	// Figure out where the user wants to store the persistent data
 	fmt.Println()
@@ -65,6 +67,16 @@ func (w *wizard) deployNode(boot bool) {
 		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
 		infos.datadir = w.readDefaultString(infos.datadir)
 	}
+	if w.conf.Genesis.Config.Ethash != nil && !boot {
+		fmt.Println()
+		if infos.ethashdir == "" {
+			fmt.Printf("Where should the ethash mining DAGs be stored on the remote machine?\n")
+			infos.ethashdir = w.readString()
+		} else {
+			fmt.Printf("Where should the ethash mining DAGs be stored on the remote machine? (default = %s)\n", infos.ethashdir)
+			infos.ethashdir = w.readDefaultString(infos.ethashdir)
+		}
+	}
 	// Figure out which port to listen on
 	fmt.Println()
 	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
@@ -91,7 +103,7 @@ func (w *wizard) deployNode(boot bool) {
 	}
 	// If the node is a miner/signer, load up needed credentials
 	if !boot {
-		if w.conf.genesis.Config.Ethash != nil {
+		if w.conf.Genesis.Config.Ethash != nil {
 			// Ethash based miners only need an etherbase to mine against
 			fmt.Println()
 			if infos.etherbase == "" {
@@ -106,7 +118,7 @@ func (w *wizard) deployNode(boot bool) {
 				fmt.Printf("What address should the miner user? (default = %s)\n", infos.etherbase)
 				infos.etherbase = w.readDefaultAddress(common.HexToAddress(infos.etherbase)).Hex()
 			}
-		} else if w.conf.genesis.Config.Clique != nil {
+		} else if w.conf.Genesis.Config.Clique != nil {
 			// If a previous signer was already set, offer to reuse it
 			if infos.keyJSON != "" {
 				if key, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
@@ -145,7 +157,13 @@ func (w *wizard) deployNode(boot bool) {
 		infos.gasPrice = w.readDefaultFloat(infos.gasPrice)
 	}
 	// Try to deploy the full node on the host
-	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos); err != nil {
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
+	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos, nocache); err != nil {
 		log.Error("Failed to deploy Ethereum node container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
@@ -156,5 +174,5 @@ func (w *wizard) deployNode(boot bool) {
 	log.Info("Waiting for node to finish booting")
 	time.Sleep(3 * time.Second)
 
-	w.networkStats(false)
+	w.networkStats()
 }

cmd/puppeth/wizard_wallet.go

@@ -0,0 +1,113 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+)
+
+// deployWallet creates a new web wallet based on some user input.
+func (w *wizard) deployWallet() {
+	// Do some sanity check before the user wastes time on input
+	if w.conf.Genesis == nil {
+		log.Error("No genesis block configured")
+		return
+	}
+	if w.conf.ethstats == "" {
+		log.Error("No ethstats server configured")
+		return
+	}
+	// Select the server to interact with
+	server := w.selectServer()
+	if server == "" {
+		return
+	}
+	client := w.servers[server]
+
+	// Retrieve any active node configurations from the server
+	infos, err := checkWallet(client, w.network)
+	if err != nil {
+		infos = &walletInfos{
+			nodePort: 30303, rpcPort: 8545, webPort: 80, webHost: client.server,
+		}
+	}
+	existed := err == nil
+
+	infos.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", "  ")
+	infos.network = w.conf.Genesis.Config.ChainId.Int64()
+
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which port should the wallet listen on? (default = %d)\n", infos.webPort)
+	infos.webPort = w.readDefaultInt(infos.webPort)
+
+	// Figure which virtual-host to deploy ethstats on
+	if infos.webHost, err = w.ensureVirtualHost(client, infos.webPort, infos.webHost); err != nil {
+		log.Error("Failed to decide on wallet host", "err", err)
+		return
+	}
+	// Figure out where the user wants to store the persistent data
+	fmt.Println()
+	if infos.datadir == "" {
+		fmt.Printf("Where should data be stored on the remote machine?\n")
+		infos.datadir = w.readString()
+	} else {
+		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		infos.datadir = w.readDefaultString(infos.datadir)
+	}
+	// Figure out which port to listen on
+	fmt.Println()
+	fmt.Printf("Which TCP/UDP port should the backing node listen on? (default = %d)\n", infos.nodePort)
+	infos.nodePort = w.readDefaultInt(infos.nodePort)
+
+	fmt.Println()
+	fmt.Printf("Which port should the backing RPC API listen on? (default = %d)\n", infos.rpcPort)
+	infos.rpcPort = w.readDefaultInt(infos.rpcPort)
+
+	// Set a proper name to report on the stats page
+	fmt.Println()
+	if infos.ethstats == "" {
+		fmt.Printf("What should the wallet be called on the stats page?\n")
+		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+	} else {
+		fmt.Printf("What should the wallet be called on the stats page? (default = %s)\n", infos.ethstats)
+		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+	}
+	// Try to deploy the wallet on the host
+	nocache := false
+	if existed {
+		fmt.Println()
+		fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n")
+		nocache = w.readDefaultString("n") != "n"
+	}
+	if out, err := deployWallet(client, w.network, w.conf.bootFull, infos, nocache); err != nil {
+		log.Error("Failed to deploy wallet container", "err", err)
+		if len(out) > 0 {
+			fmt.Printf("%s\n", out)
+		}
+		return
+	}
+	// All ok, run a network scan to pick any changes up
+	log.Info("Waiting for node to finish booting")
+	time.Sleep(3 * time.Second)
+
+	w.networkStats()
+}

cmd/rlpdump/main.go

@@ -51,7 +51,7 @@ func main() {
 	var r io.Reader
 	switch {
 	case *hexMode != "":
-		data, err := hex.DecodeString(*hexMode)
+		data, err := hex.DecodeString(strings.TrimPrefix(*hexMode, "0x"))
 		if err != nil {
 			die(err)
 		}

cmd/swarm/manifest.go

@@ -30,6 +30,8 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+const bzzManifestJSON = "application/bzz-manifest+json"
+
 func add(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 3 {
@@ -145,7 +147,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		if path == entry.Path {
 			utils.Fatalf("Path %s already present, not adding anything", path)
 		} else {
-			if entry.ContentType == "application/bzz-manifest+json" {
+			if entry.ContentType == bzzManifestJSON {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
 					longestPathEntry = entry
@@ -207,7 +209,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		if path == entry.Path {
 			newEntry = entry
 		} else {
-			if entry.ContentType == "application/bzz-manifest+json" {
+			if entry.ContentType == bzzManifestJSON {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
 					longestPathEntry = entry
@@ -281,7 +283,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		if path == entry.Path {
 			entryToRemove = entry
 		} else {
-			if entry.ContentType == "application/bzz-manifest+json" {
+			if entry.ContentType == bzzManifestJSON {
 				prfxlen := strings.HasPrefix(path, entry.Path)
 				if prfxlen && len(path) > len(longestPathEntry.Path) {
 					longestPathEntry = entry

cmd/utils/flags.go

@@ -31,11 +31,14 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/accounts"
 	"github.com/EthereumCommonwealth/go-callisto/accounts/keystore"
 	"github.com/EthereumCommonwealth/go-callisto/common"
+	"github.com/EthereumCommonwealth/go-callisto/consensus"
+	"github.com/EthereumCommonwealth/go-callisto/consensus/clique"
 	"github.com/EthereumCommonwealth/go-callisto/consensus/ethash"
 	"github.com/EthereumCommonwealth/go-callisto/core"
 	"github.com/EthereumCommonwealth/go-callisto/core/state"
 	"github.com/EthereumCommonwealth/go-callisto/core/vm"
 	"github.com/EthereumCommonwealth/go-callisto/crypto"
+	"github.com/EthereumCommonwealth/go-callisto/dashboard"
 	"github.com/EthereumCommonwealth/go-callisto/eth"
 	"github.com/EthereumCommonwealth/go-callisto/eth/downloader"
 	"github.com/EthereumCommonwealth/go-callisto/eth/gasprice"
@@ -135,9 +138,13 @@ var (
 		Name:  "rinkeby",
 		Usage: "Rinkeby network: pre-configured proof-of-authority test network",
 	}
-	DevModeFlag = cli.BoolFlag{
+	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
-		Usage: "Developer mode: pre-configured private network with several debugging flags",
+		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
+	}
+	DeveloperPeriodFlag = cli.IntFlag{
+		Name:  "dev.period",
+		Usage: "Block period to use in developer mode (0 = mine only if transaction pending)",
 	}
 	IdentityFlag = cli.StringFlag{
 		Name:  "identity",
@@ -177,6 +184,31 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	// Dashboard settings
+	DashboardEnabledFlag = cli.BoolFlag{
+		Name:  "dashboard",
+		Usage: "Enable the dashboard",
+	}
+	DashboardAddrFlag = cli.StringFlag{
+		Name:  "dashboard.addr",
+		Usage: "Dashboard listening interface",
+		Value: dashboard.DefaultConfig.Host,
+	}
+	DashboardPortFlag = cli.IntFlag{
+		Name:  "dashboard.host",
+		Usage: "Dashboard listening port",
+		Value: dashboard.DefaultConfig.Port,
+	}
+	DashboardRefreshFlag = cli.DurationFlag{
+		Name:  "dashboard.refresh",
+		Usage: "Dashboard metrics collection refresh rate",
+		Value: dashboard.DefaultConfig.Refresh,
+	}
+	DashboardAssetsFlag = cli.StringFlag{
+		Name:  "dashboard.assets",
+		Usage: "Developer flag to serve the dashboard from the local file system",
+		Value: dashboard.DefaultConfig.Assets,
+	}
 	// Ethash settings
 	EthashCacheDirFlag = DirectoryFlag{
 		Name:  "ethash.cachedir",
@@ -185,27 +217,27 @@ var (
 	EthashCachesInMemoryFlag = cli.IntFlag{
 		Name:  "ethash.cachesinmem",
 		Usage: "Number of recent ethash caches to keep in memory (16MB each)",
-		Value: eth.DefaultConfig.EthashCachesInMem,
+		Value: eth.DefaultConfig.Ethash.CachesInMem,
 	}
 	EthashCachesOnDiskFlag = cli.IntFlag{
 		Name:  "ethash.cachesondisk",
 		Usage: "Number of recent ethash caches to keep on disk (16MB each)",
-		Value: eth.DefaultConfig.EthashCachesOnDisk,
+		Value: eth.DefaultConfig.Ethash.CachesOnDisk,
 	}
 	EthashDatasetDirFlag = DirectoryFlag{
 		Name:  "ethash.dagdir",
 		Usage: "Directory to store the ethash mining DAGs (default = inside home folder)",
-		Value: DirectoryString{eth.DefaultConfig.EthashDatasetDir},
+		Value: DirectoryString{eth.DefaultConfig.Ethash.DatasetDir},
 	}
 	EthashDatasetsInMemoryFlag = cli.IntFlag{
 		Name:  "ethash.dagsinmem",
 		Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)",
-		Value: eth.DefaultConfig.EthashDatasetsInMem,
+		Value: eth.DefaultConfig.Ethash.DatasetsInMem,
 	}
 	EthashDatasetsOnDiskFlag = cli.IntFlag{
 		Name:  "ethash.dagsondisk",
 		Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)",
-		Value: eth.DefaultConfig.EthashDatasetsOnDisk,
+		Value: eth.DefaultConfig.Ethash.DatasetsOnDisk,
 	}
 	// Transaction pool settings
 	TxPoolNoLocalsFlag = cli.BoolFlag{
@@ -552,6 +584,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 		urls = params.TestnetBootnodes
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		urls = params.RinkebyBootnodes
+	case cfg.BootstrapNodes != nil:
+		return // already set, don't apply defaults.
 	}
 
 	cfg.BootstrapNodes = make([]*discover.Node, 0, len(urls))
@@ -794,7 +828,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 		cfg.NetRestrict = list
 	}
 
-	if ctx.GlobalBool(DevModeFlag.Name) {
+	if ctx.GlobalBool(DeveloperFlag.Name) {
 		// --dev mode can't use p2p networking.
 		cfg.MaxPeers = 0
 		cfg.ListenAddr = ":0"
@@ -815,8 +849,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	switch {
 	case ctx.GlobalIsSet(DataDirFlag.Name):
 		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
-	case ctx.GlobalBool(DevModeFlag.Name):
+	case ctx.GlobalBool(DeveloperFlag.Name):
-		cfg.DataDir = filepath.Join(os.TempDir(), "ethereum_dev_mode")
+		cfg.DataDir = "" // unless explicitly requested, use memory databases
 	case ctx.GlobalBool(TestnetFlag.Name):
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
 	case ctx.GlobalBool(RinkebyFlag.Name):
@@ -878,34 +912,60 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) {
 
 func setEthash(ctx *cli.Context, cfg *eth.Config) {
 	if ctx.GlobalIsSet(EthashCacheDirFlag.Name) {
-		cfg.EthashCacheDir = ctx.GlobalString(EthashCacheDirFlag.Name)
+		cfg.Ethash.CacheDir = ctx.GlobalString(EthashCacheDirFlag.Name)
 	}
 	if ctx.GlobalIsSet(EthashDatasetDirFlag.Name) {
-		cfg.EthashDatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name)
+		cfg.Ethash.DatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name)
 	}
 	if ctx.GlobalIsSet(EthashCachesInMemoryFlag.Name) {
-		cfg.EthashCachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name)
+		cfg.Ethash.CachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name)
 	}
 	if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) {
-		cfg.EthashCachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
+		cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
 	}
 	if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) {
-		cfg.EthashDatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name)
+		cfg.Ethash.DatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name)
 	}
 	if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) {
-		cfg.EthashDatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name)
+		cfg.Ethash.DatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name)
 	}
 }
 
-func checkExclusive(ctx *cli.Context, flags ...cli.Flag) {
+// checkExclusive verifies that only a single isntance of the provided flags was
+// set by the user. Each flag might optionally be followed by a string type to
+// specialize it further.
+func checkExclusive(ctx *cli.Context, args ...interface{}) {
 	set := make([]string, 0, 1)
-	for _, flag := range flags {
+	for i := 0; i < len(args); i++ {
+		// Make sure the next argument is a flag and skip if not set
+		flag, ok := args[i].(cli.Flag)
+		if !ok {
+			panic(fmt.Sprintf("invalid argument, not cli.Flag type: %T", args[i]))
+		}
+		// Check if next arg extends current and expand its name if so
+		name := flag.GetName()
+
+		if i+1 < len(args) {
+			switch option := args[i+1].(type) {
+			case string:
+				// Extended flag, expand the name and shift the arguments
+				if ctx.GlobalString(flag.GetName()) == option {
+					name += "=" + option
+				}
+				i++
+
+			case cli.Flag:
+			default:
+				panic(fmt.Sprintf("invalid argument, not cli.Flag or string extension: %T", args[i+1]))
+			}
+		}
+		// Mark the flag if it's set
 		if ctx.GlobalIsSet(flag.GetName()) {
-			set = append(set, "--"+flag.GetName())
+			set = append(set, "--"+name)
 		}
 	}
 	if len(set) > 1 {
-		Fatalf("flags %v can't be used at the same time", strings.Join(set, ", "))
+		Fatalf("Flags %v can't be used at the same time", strings.Join(set, ", "))
 	}
 }
 
@@ -922,8 +982,10 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	// Avoid conflicting network flags
-	checkExclusive(ctx, DevModeFlag, TestnetFlag, RinkebyFlag)
+	checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag)
 	checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag)
+	checkExclusive(ctx, LightServFlag, LightModeFlag)
+	checkExclusive(ctx, LightServFlag, SyncModeFlag, "light")
 
 	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 	setEtherbase(ctx, ks, cfg)
@@ -983,20 +1045,44 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 			cfg.NetworkId = 4
 		}
 		cfg.Genesis = core.DefaultRinkebyGenesisBlock()
-	case ctx.GlobalBool(DevModeFlag.Name):
+	case ctx.GlobalBool(DeveloperFlag.Name):
-		cfg.Genesis = core.DevGenesisBlock()
+		// Create new developer account or reuse existing one
-		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
+		var (
-			cfg.GasPrice = new(big.Int)
+			developer accounts.Account
+			err       error
+		)
+		if accs := ks.Accounts(); len(accs) > 0 {
+			developer = ks.Accounts()[0]
+		} else {
+			developer, err = ks.NewAccount("")
+			if err != nil {
+				Fatalf("Failed to create developer account: %v", err)
+			}
 		}
-		cfg.PowTest = true
+		if err := ks.Unlock(developer, ""); err != nil {
-	}
+			Fatalf("Failed to unlock developer account: %v", err)
+		}
+		log.Info("Using developer account", "address", developer.Address)
 
+		cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address)
+		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
+			cfg.GasPrice = big.NewInt(1)
+		}
+	}
 	// TODO(fjl): move trie cache generations into config
 	if gen := ctx.GlobalInt(TrieCacheGenFlag.Name); gen > 0 {
 		state.MaxTrieCacheGen = uint16(gen)
 	}
 }
 
+// SetDashboardConfig applies dashboard related command line flags to the config.
+func SetDashboardConfig(ctx *cli.Context, cfg *dashboard.Config) {
+	cfg.Host = ctx.GlobalString(DashboardAddrFlag.Name)
+	cfg.Port = ctx.GlobalInt(DashboardPortFlag.Name)
+	cfg.Refresh = ctx.GlobalDuration(DashboardRefreshFlag.Name)
+	cfg.Assets = ctx.GlobalString(DashboardAssetsFlag.Name)
+}
+
 // RegisterEthService adds an Ethereum client to the stack.
 func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 	var err error
@@ -1019,6 +1105,13 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) {
 	}
 }
 
+// RegisterDashboardService adds a dashboard to the stack.
+func RegisterDashboardService(stack *node.Node, cfg *dashboard.Config) {
+	stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		return dashboard.New(cfg)
+	})
+}
+
 // RegisterShhService configures Whisper and adds it to the given node.
 func RegisterShhService(stack *node.Node, cfg *whisper.Config) {
 	if err := stack.Register(func(n *node.ServiceContext) (node.Service, error) {
@@ -1075,8 +1168,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 		genesis = core.DefaultTestnetGenesisBlock()
 	case ctx.GlobalBool(RinkebyFlag.Name):
 		genesis = core.DefaultRinkebyGenesisBlock()
-	case ctx.GlobalBool(DevModeFlag.Name):
+	case ctx.GlobalBool(DeveloperFlag.Name):
-		genesis = core.DevGenesisBlock()
+		Fatalf("Developer chains are ephemeral")
 	}
 	return genesis
 }
@@ -1086,17 +1179,26 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	var err error
 	chainDb = MakeChainDatabase(ctx, stack)
 
-	engine := ethash.NewFaker()
-	if !ctx.GlobalBool(FakePoWFlag.Name) {
-		engine = ethash.New(
-			stack.ResolvePath(eth.DefaultConfig.EthashCacheDir), eth.DefaultConfig.EthashCachesInMem, eth.DefaultConfig.EthashCachesOnDisk,
-			stack.ResolvePath(eth.DefaultConfig.EthashDatasetDir), eth.DefaultConfig.EthashDatasetsInMem, eth.DefaultConfig.EthashDatasetsOnDisk,
-		)
-	}
 	config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
 	if err != nil {
 		Fatalf("%v", err)
 	}
+	var engine consensus.Engine
+	if config.Clique != nil {
+		engine = clique.New(config.Clique, chainDb)
+	} else {
+		engine = ethash.NewFaker()
+		if !ctx.GlobalBool(FakePoWFlag.Name) {
+			engine = ethash.New(ethash.Config{
+				CacheDir:       stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
+				CachesInMem:    eth.DefaultConfig.Ethash.CachesInMem,
+				CachesOnDisk:   eth.DefaultConfig.Ethash.CachesOnDisk,
+				DatasetDir:     stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
+				DatasetsInMem:  eth.DefaultConfig.Ethash.DatasetsInMem,
+				DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
+			})
+		}
+	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
 	if err != nil {

common/bytes.go

@@ -35,12 +35,11 @@ func FromHex(s string) []byte {
 		if s[0:2] == "0x" || s[0:2] == "0X" {
 			s = s[2:]
 		}
-		if len(s)%2 == 1 {
-			s = "0" + s
-		}
-		return Hex2Bytes(s)
 	}
-	return nil
+	if len(s)%2 == 1 {
+		s = "0" + s
+	}
+	return Hex2Bytes(s)
 }
 
 // Copy bytes

common/bytes_test.go

@@ -74,7 +74,7 @@ func TestFromHex(t *testing.T) {
 	expected := []byte{1}
 	result := FromHex(input)
 	if !bytes.Equal(expected, result) {
-		t.Errorf("Expected % x got % x", expected, result)
+		t.Errorf("Expected %x got %x", expected, result)
 	}
 }
 
@@ -83,6 +83,15 @@ func TestFromHexOddLength(t *testing.T) {
 	expected := []byte{1}
 	result := FromHex(input)
 	if !bytes.Equal(expected, result) {
-		t.Errorf("Expected % x got % x", expected, result)
+		t.Errorf("Expected %x got %x", expected, result)
+	}
+}
+
+func TestNoPrefixShortHexOddLength(t *testing.T) {
+	input := "1"
+	expected := []byte{1}
+	result := FromHex(input)
+	if !bytes.Equal(expected, result) {
+		t.Errorf("Expected %x got %x", expected, result)
 	}
 }

common/hexutil/hexutil.go

@@ -53,9 +53,7 @@ var (
 
 type decError struct{ msg string }
 
-func (err decError) Error() string {
+func (err decError) Error() string { return err.msg }
-	return string(err.msg)
-}
 
 // Decode decodes a hex string with 0x prefix.
 func Decode(input string) ([]byte, error) {

common/hexutil/json.go

@@ -223,7 +223,7 @@ func (b *Uint64) UnmarshalText(input []byte) error {
 			return ErrSyntax
 		}
 		dec *= 16
-		dec += uint64(nib)
+		dec += nib
 	}
 	*b = Uint64(dec)
 	return nil

consensus/clique/clique.go

@@ -29,6 +29,7 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/common/hexutil"
 	"github.com/EthereumCommonwealth/go-callisto/consensus"
+	"github.com/EthereumCommonwealth/go-callisto/consensus/misc"
 	"github.com/EthereumCommonwealth/go-callisto/core/state"
 	"github.com/EthereumCommonwealth/go-callisto/core/types"
 	"github.com/EthereumCommonwealth/go-callisto/crypto"
@@ -124,6 +125,11 @@ var (
 
 	// errUnauthorized is returned if a header is signed by a non-authorized entity.
 	errUnauthorized = errors.New("unauthorized")
+
+	// errWaitTransactions is returned if an empty block is attempted to be sealed
+	// on an instant chain (0 second period). It's important to refuse these as the
+	// block reward is zero, so an empty block just bloats the chain... fast.
+	errWaitTransactions = errors.New("waiting for transactions")
 )
 
 // SignerFn is a signer callback function to request a hash to be signed by a
@@ -210,9 +216,6 @@ func New(config *params.CliqueConfig, db ethdb.Database) *Clique {
 	if conf.Epoch == 0 {
 		conf.Epoch = epochLength
 	}
-	if conf.Period == 0 {
-		conf.Period = blockPeriod
-	}
 	// Allocate the snapshot caches and create the engine
 	recents, _ := lru.NewARC(inmemorySnapshots)
 	signatures, _ := lru.NewARC(inmemorySignatures)
@@ -313,6 +316,10 @@ func (c *Clique) verifyHeader(chain consensus.ChainReader, header *types.Header,
 			return errInvalidDifficulty
 		}
 	}
+	// If all checks passed, validate any special fields for hard forks
+	if err := misc.VerifyForkHashes(chain.Config(), header, false); err != nil {
+		return err
+	}
 	// All basic checks passed, verify cascading fields
 	return c.verifyCascadingFields(chain, header, parents)
 }
@@ -594,6 +601,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 	if number == 0 {
 		return nil, errUnknownBlock
 	}
+	// For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing)
+	if c.config.Period == 0 && len(block.Transactions()) == 0 {
+		return nil, errWaitTransactions
+	}
 	// Don't hold the signer fields for the entire sealing procedure
 	c.lock.RLock()
 	signer, signFn := c.signer, c.signFn

consensus/clique/snapshot_test.go

@@ -74,7 +74,7 @@ type testerChainReader struct {
 	db ethdb.Database
 }
 
-func (r *testerChainReader) Config() *params.ChainConfig                 { panic("not supported") }
+func (r *testerChainReader) Config() *params.ChainConfig                 { return params.AllCliqueProtocolChanges }
 func (r *testerChainReader) CurrentHeader() *types.Header                { panic("not supported") }
 func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
 func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block   { panic("not supported") }

consensus/ethash/algorithm.go

@@ -102,7 +102,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) {
 	header.Cap *= 4
 	cache := *(*[]byte)(unsafe.Pointer(&header))
 
-	// Calculate the number of thoretical rows (we'll store in one buffer nonetheless)
+	// Calculate the number of theoretical rows (we'll store in one buffer nonetheless)
 	size := uint64(len(cache))
 	rows := int(size) / hashBytes
 
@@ -187,7 +187,7 @@ func fnvHash(mix []uint32, data []uint32) {
 // generateDatasetItem combines data from 256 pseudorandomly selected cache nodes,
 // and hashes that to compute a single dataset node.
 func generateDatasetItem(cache []uint32, index uint32, keccak512 hasher) []byte {
-	// Calculate the number of thoretical rows (we use one buffer nonetheless)
+	// Calculate the number of theoretical rows (we use one buffer nonetheless)
 	rows := uint32(len(cache) / hashWords)
 
 	// Initialize the mix
@@ -287,7 +287,7 @@ func generateDataset(dest []uint32, epoch uint64, cache []uint32) {
 // hashimoto aggregates data from the full dataset in order to produce our final
 // value for a particular header hash and nonce.
 func hashimoto(hash []byte, nonce uint64, size uint64, lookup func(index uint32) []uint32) ([]byte, []byte) {
-	// Calculate the number of thoretical rows (we use one buffer nonetheless)
+	// Calculate the number of theoretical rows (we use one buffer nonetheless)
 	rows := uint32(size / mixBytes)
 
 	// Combine header+nonce into a 64 byte seed

consensus/ethash/algorithm_go1.8.go

@@ -31,7 +31,7 @@ func cacheSize(block uint64) uint64 {
 		return cacheSizes[epoch]
 	}
 	// No known cache size, calculate manually (sanity branch only)
-	size := uint64(cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes)
+	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
 	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * hashBytes
 	}
@@ -49,7 +49,7 @@ func datasetSize(block uint64) uint64 {
 		return datasetSizes[epoch]
 	}
 	// No known dataset size, calculate manually (sanity branch only)
-	size := uint64(datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes)
+	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
 	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
 		size -= 2 * mixBytes
 	}

consensus/ethash/algorithm_test.go

@@ -703,8 +703,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 
 		go func(idx int) {
 			defer pend.Done()
-
+			ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal})
-			ethash := New(cachedir, 0, 1, "", 0, 0)
 			if err := ethash.VerifySeal(nil, block.Header()); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)
 			}

consensus/ethash/consensus.go

@@ -36,10 +36,10 @@ import (
 
 // Ethash proof-of-work protocol constants.
 var (
-	callistoBlockReward, _ = new(big.Int).SetString("10000000000000000000", 10) // Block reward in wei for successfully mining a block
+	callistoBlockReward, _          = new(big.Int).SetString("10000000000000000000", 10) // Block reward in wei for successfully mining a block
-	frontierBlockReward  *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
+	frontierBlockReward    *big.Int = big.NewInt(5e+18)                                  // Block reward in wei for successfully mining a block
-	byzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
+	byzantiumBlockReward   *big.Int = big.NewInt(3e+18)                                  // Block reward in wei for successfully mining a block upward from Byzantium
-	maxUncles                     = 2                 // Maximum number of uncles allowed in a single block
+	maxUncles                       = 2                                                  // Maximum number of uncles allowed in a single block
 )
 
 // Various error messages to mark blocks invalid. These should be private to
@@ -69,7 +69,7 @@ func (ethash *Ethash) Author(header *types.Header) (common.Address, error) {
 // stock Ethereum ethash engine.
 func (ethash *Ethash) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error {
 	// If we're running a full engine faking, accept any input as valid
-	if ethash.fakeFull {
+	if ethash.config.PowMode == ModeFullFake {
 		return nil
 	}
 	// Short circuit if the header is known, or it's parent not
@@ -90,7 +90,7 @@ func (ethash *Ethash) VerifyHeader(chain consensus.ChainReader, header *types.He
 // a results channel to retrieve the async verifications.
 func (ethash *Ethash) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
 	// If we're running a full engine faking, accept any input as valid
-	if ethash.fakeFull || len(headers) == 0 {
+	if ethash.config.PowMode == ModeFullFake || len(headers) == 0 {
 		abort, results := make(chan struct{}), make(chan error, len(headers))
 		for i := 0; i < len(headers); i++ {
 			results <- nil
@@ -170,7 +170,7 @@ func (ethash *Ethash) verifyHeaderWorker(chain consensus.ChainReader, headers []
 // rules of the stock Ethereum ethash engine.
 func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
 	// If we're running a full engine faking, accept any input as valid
-	if ethash.fakeFull {
+	if ethash.config.PowMode == ModeFullFake {
 		return nil
 	}
 	// Verify that there are at most 2 uncles included in this block
@@ -456,7 +456,7 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int {
 // the PoW difficulty requirements.
 func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Header) error {
 	// If we're running a fake PoW, accept any seal as valid
-	if ethash.fakeMode {
+	if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake {
 		time.Sleep(ethash.fakeDelay)
 		if ethash.fakeFail == header.Number.Uint64() {
 			return errInvalidPoW
@@ -481,7 +481,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	cache := ethash.cache(number)
 
 	size := datasetSize(number)
-	if ethash.tester {
+	if ethash.config.PowMode == ModeTest {
 		size = 32 * 1024
 	}
 	digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
@@ -556,7 +556,6 @@ func AccumulateRewards(config *params.ChainConfig, state *state.StateDB, header
 	state.AddBalance(common.HexToAddress("SET ADDRESS IN SETTINGS"), treasuryReward)
 }
 
-
 // getTreasuryPercent computes the current block reward's percent will be for
 // dev subsidy.
 func getTreasuryPercent(header *types.Header) int64 {

consensus/ethash/ethash.go

@@ -45,7 +45,7 @@ var (
 	maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = New("", 3, 0, "", 1, 0)
+	sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal})
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -320,15 +320,32 @@ func MakeDataset(block uint64, dir string) {
 	d.release()
 }
 
+// Mode defines the type and amount of PoW verification an ethash engine makes.
+type Mode uint
+
+const (
+	ModeNormal Mode = iota
+	ModeShared
+	ModeTest
+	ModeFake
+	ModeFullFake
+)
+
+// Config are the configuration parameters of the ethash.
+type Config struct {
+	CacheDir       string
+	CachesInMem    int
+	CachesOnDisk   int
+	DatasetDir     string
+	DatasetsInMem  int
+	DatasetsOnDisk int
+	PowMode        Mode
+}
+
 // Ethash is a consensus engine based on proot-of-work implementing the ethash
 // algorithm.
 type Ethash struct {
-	cachedir     string // Data directory to store the verification caches
+	config Config
-	cachesinmem  int    // Number of caches to keep in memory
-	cachesondisk int    // Number of caches to keep on disk
-	dagdir       string // Data directory to store full mining datasets
-	dagsinmem    int    // Number of mining datasets to keep in memory
-	dagsondisk   int    // Number of mining datasets to keep on disk
 
 	caches   map[uint64]*cache   // In memory caches to avoid regenerating too often
 	fcache   *cache              // Pre-generated cache for the estimated future epoch
@@ -342,10 +359,7 @@ type Ethash struct {
 	hashrate metrics.Meter // Meter tracking the average hashrate
 
 	// The fields below are hooks for testing
-	tester    bool          // Flag whether to use a smaller test dataset
 	shared    *Ethash       // Shared PoW verifier to avoid cache regeneration
-	fakeMode  bool          // Flag whether to disable PoW checking
-	fakeFull  bool          // Flag whether to disable all consensus rules
 	fakeFail  uint64        // Block number which fails PoW check even in fake mode
 	fakeDelay time.Duration // Time delay to sleep for before returning from verify
 
@@ -353,28 +367,23 @@ type Ethash struct {
 }
 
 // New creates a full sized ethash PoW scheme.
-func New(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) *Ethash {
+func New(config Config) *Ethash {
-	if cachesinmem <= 0 {
+	if config.CachesInMem <= 0 {
-		log.Warn("One ethash cache must always be in memory", "requested", cachesinmem)
+		log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem)
-		cachesinmem = 1
+		config.CachesInMem = 1
 	}
-	if cachedir != "" && cachesondisk > 0 {
+	if config.CacheDir != "" && config.CachesOnDisk > 0 {
-		log.Info("Disk storage enabled for ethash caches", "dir", cachedir, "count", cachesondisk)
+		log.Info("Disk storage enabled for ethash caches", "dir", config.CacheDir, "count", config.CachesOnDisk)
 	}
-	if dagdir != "" && dagsondisk > 0 {
+	if config.DatasetDir != "" && config.DatasetsOnDisk > 0 {
-		log.Info("Disk storage enabled for ethash DAGs", "dir", dagdir, "count", dagsondisk)
+		log.Info("Disk storage enabled for ethash DAGs", "dir", config.DatasetDir, "count", config.DatasetsOnDisk)
 	}
 	return &Ethash{
-		cachedir:     cachedir,
+		config:   config,
-		cachesinmem:  cachesinmem,
+		caches:   make(map[uint64]*cache),
-		cachesondisk: cachesondisk,
+		datasets: make(map[uint64]*dataset),
-		dagdir:       dagdir,
+		update:   make(chan struct{}),
-		dagsinmem:    dagsinmem,
+		hashrate: metrics.NewMeter(),
-		dagsondisk:   dagsondisk,
-		caches:       make(map[uint64]*cache),
-		datasets:     make(map[uint64]*dataset),
-		update:       make(chan struct{}),
-		hashrate:     metrics.NewMeter(),
 	}
 }
 
@@ -382,12 +391,14 @@ func New(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinme
 // purposes.
 func NewTester() *Ethash {
 	return &Ethash{
-		cachesinmem: 1,
+		config: Config{
-		caches:      make(map[uint64]*cache),
+			CachesInMem: 1,
-		datasets:    make(map[uint64]*dataset),
+			PowMode:     ModeTest,
-		tester:      true,
+		},
-		update:      make(chan struct{}),
+		caches:   make(map[uint64]*cache),
-		hashrate:    metrics.NewMeter(),
+		datasets: make(map[uint64]*dataset),
+		update:   make(chan struct{}),
+		hashrate: metrics.NewMeter(),
 	}
 }
 
@@ -395,27 +406,45 @@ func NewTester() *Ethash {
 // all blocks' seal as valid, though they still have to conform to the Ethereum
 // consensus rules.
 func NewFaker() *Ethash {
-	return &Ethash{fakeMode: true}
+	return &Ethash{
+		config: Config{
+			PowMode: ModeFake,
+		},
+	}
 }
 
 // NewFakeFailer creates a ethash consensus engine with a fake PoW scheme that
 // accepts all blocks as valid apart from the single one specified, though they
 // still have to conform to the Ethereum consensus rules.
 func NewFakeFailer(fail uint64) *Ethash {
-	return &Ethash{fakeMode: true, fakeFail: fail}
+	return &Ethash{
+		config: Config{
+			PowMode: ModeFake,
+		},
+		fakeFail: fail,
+	}
 }
 
 // NewFakeDelayer creates a ethash consensus engine with a fake PoW scheme that
 // accepts all blocks as valid, but delays verifications by some time, though
 // they still have to conform to the Ethereum consensus rules.
 func NewFakeDelayer(delay time.Duration) *Ethash {
-	return &Ethash{fakeMode: true, fakeDelay: delay}
+	return &Ethash{
+		config: Config{
+			PowMode: ModeFake,
+		},
+		fakeDelay: delay,
+	}
 }
 
 // NewFullFaker creates an ethash consensus engine with a full fake scheme that
 // accepts all blocks as valid, without checking any consensus rules whatsoever.
 func NewFullFaker() *Ethash {
-	return &Ethash{fakeMode: true, fakeFull: true}
+	return &Ethash{
+		config: Config{
+			PowMode: ModeFullFake,
+		},
+	}
 }
 
 // NewShared creates a full sized ethash PoW shared between all requesters running
@@ -436,7 +465,7 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 	current, future := ethash.caches[epoch], (*cache)(nil)
 	if current == nil {
 		// No in-memory cache, evict the oldest if the cache limit was reached
-		for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.cachesinmem {
+		for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.config.CachesInMem {
 			var evict *cache
 			for _, cache := range ethash.caches {
 				if evict == nil || evict.used.After(cache.used) {
@@ -473,7 +502,7 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 	ethash.lock.Unlock()
 
 	// Wait for generation finish, bump the timestamp and finalize the cache
-	current.generate(ethash.cachedir, ethash.cachesondisk, ethash.tester)
+	current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
 
 	current.lock.Lock()
 	current.used = time.Now()
@@ -481,7 +510,7 @@ func (ethash *Ethash) cache(block uint64) []uint32 {
 
 	// If we exhausted the future cache, now's a good time to regenerate it
 	if future != nil {
-		go future.generate(ethash.cachedir, ethash.cachesondisk, ethash.tester)
+		go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
 	}
 	return current.cache
 }
@@ -498,7 +527,7 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 	current, future := ethash.datasets[epoch], (*dataset)(nil)
 	if current == nil {
 		// No in-memory dataset, evict the oldest if the dataset limit was reached
-		for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.dagsinmem {
+		for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.config.DatasetsInMem {
 			var evict *dataset
 			for _, dataset := range ethash.datasets {
 				if evict == nil || evict.used.After(dataset.used) {
@@ -536,7 +565,7 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 	ethash.lock.Unlock()
 
 	// Wait for generation finish, bump the timestamp and finalize the cache
-	current.generate(ethash.dagdir, ethash.dagsondisk, ethash.tester)
+	current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
 
 	current.lock.Lock()
 	current.used = time.Now()
@@ -544,7 +573,7 @@ func (ethash *Ethash) dataset(block uint64) []uint32 {
 
 	// If we exhausted the future dataset, now's a good time to regenerate it
 	if future != nil {
-		go future.generate(ethash.dagdir, ethash.dagsondisk, ethash.tester)
+		go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
 	}
 	return current.dataset
 }

consensus/ethash/sealer.go

@@ -34,7 +34,7 @@ import (
 // the block's difficulty requirements.
 func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
 	// If we're running a fake PoW, simply return a 0 nonce immediately
-	if ethash.fakeMode {
+	if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake {
 		header := block.Header()
 		header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{}
 		return block.WithSeal(header), nil

console/console.go

@@ -47,7 +47,7 @@ const HistoryFile = "history"
 // DefaultPrompt is the default prompt line prefix to use for user input querying.
 const DefaultPrompt = "> "
 
-// Config is te collection of configurations to fine tune the behavior of the
+// Config is the collection of configurations to fine tune the behavior of the
 // JavaScript console.
 type Config struct {
 	DataDir  string       // Data directory to store the console history at
@@ -238,7 +238,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str
 	// E.g. in case of nested lines eth.getBalance(eth.coinb<tab><tab>
 	start := pos - 1
 	for ; start > 0; start-- {
-		// Skip all methods and namespaces (i.e. including te dot)
+		// Skip all methods and namespaces (i.e. including the dot)
 		if line[start] == '.' || (line[start] >= 'a' && line[start] <= 'z') || (line[start] >= 'A' && line[start] <= 'Z') {
 			continue
 		}

console/console_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/EthereumCommonwealth/go-callisto/common"
+	"github.com/EthereumCommonwealth/go-callisto/consensus/ethash"
 	"github.com/EthereumCommonwealth/go-callisto/core"
 	"github.com/EthereumCommonwealth/go-callisto/eth"
 	"github.com/EthereumCommonwealth/go-callisto/internal/jsre"
@@ -94,9 +95,11 @@ func newTester(t *testing.T, confOverride func(*eth.Config)) *tester {
 		t.Fatalf("failed to create node: %v", err)
 	}
 	ethConf := &eth.Config{
-		Genesis:   core.DevGenesisBlock(),
+		Genesis:   core.DeveloperGenesisBlock(15, common.Address{}),
 		Etherbase: common.HexToAddress(testAddress),
-		PowTest:   true,
+		Ethash: ethash.Config{
+			PowMode: ethash.ModeTest,
+		},
 	}
 	if confOverride != nil {
 		confOverride(ethConf)

contracts/chequebook/contract/chequebook.sol

@@ -1,7 +1,9 @@
-import "mortal";
+pragma solidity ^0.4.18;
+
+import "https://github.com/ethereum/solidity/std/mortal.sol";
 
 /// @title Chequebook for Ethereum micropayments
-/// @author Daniel A. Nagy <daniel@ethdev.com>
+/// @author Daniel A. Nagy <daniel@ethereum.org>
 contract chequebook is mortal {
     // Cumulative paid amount in wei to each beneficiary
     mapping (address => uint256) public sent;
@@ -21,26 +23,23 @@ contract chequebook is mortal {
         uint8 sig_v, bytes32 sig_r, bytes32 sig_s) {
         // Check if the cheque is old.
         // Only cheques that are more recent than the last cashed one are considered.
-        if(amount <= sent[beneficiary]) return;
+        require(amount > sent[beneficiary]);
         // Check the digital signature of the cheque.
-        bytes32 hash = sha3(address(this), beneficiary, amount);
+        bytes32 hash = keccak256(address(this), beneficiary, amount);
-        if(owner != ecrecover(hash, sig_v, sig_r, sig_s)) return;
+        require(owner == ecrecover(hash, sig_v, sig_r, sig_s));
         // Attempt sending the difference between the cumulative amount on the cheque
         // and the cumulative amount on the last cashed cheque to beneficiary.
         uint256 diff = amount - sent[beneficiary];
         if (diff <= this.balance) {
 	    // update the cumulative amount before sending
             sent[beneficiary] = amount;
-            if (!beneficiary.send(diff)) {
+            beneficiary.transfer(diff);
-                // Upon failure to execute send, revert everything
-                throw;
-            }
         } else {
             // Upon failure, punish owner for writing a bounced cheque.
             // owner.sendToDebtorsPrison();
             Overdraft(owner);
             // Compensate beneficiary.
-            suicide(beneficiary);
+            selfdestruct(beneficiary);
         }
     }
 }

contracts/release/contract.sol

@@ -77,7 +77,7 @@ contract ReleaseOracle {
     }
   }
 
-  // signers is an accessor method to retrieve all te signers (public accessor
+  // signers is an accessor method to retrieve all the signers (public accessor
   // generates an indexed one, not a retrieve-all version).
   function signers() constant returns(address[]) {
     return voters;

core/blockchain.go

@@ -818,7 +818,12 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
 	// If the total difficulty is higher than our known, add it to the canonical chain
 	// Second clause in the if statement reduces the vulnerability to selfish mining.
 	// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
-	if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) {
+	reorg := externTd.Cmp(localTd) > 0
+	if !reorg && externTd.Cmp(localTd) == 0 {
+		// Split same-difficulty blocks by number, then at random
+		reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5)
+	}
+	if reorg {
 		// Reorganise the chain if the parent is not the head block
 		if block.ParentHash() != bc.currentBlock.Hash() {
 			if err := bc.reorg(bc.currentBlock, block); err != nil {

core/bloombits/matcher.go

@@ -18,6 +18,7 @@ package bloombits
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"math"
 	"sort"
@@ -56,10 +57,16 @@ type partialMatches struct {
 // Retrieval represents a request for retrieval task assignments for a given
 // bit with the given number of fetch elements, or a response for such a request.
 // It can also have the actual results set to be used as a delivery data struct.
+//
+// The contest and error fields are used by the light client to terminate matching
+// early if an error is enountered on some path of the pipeline.
 type Retrieval struct {
 	Bit      uint
 	Sections []uint64
 	Bitsets  [][]byte
+
+	Context context.Context
+	Error   error
 }
 
 // Matcher is a pipelined system of schedulers and logic matchers which perform
@@ -137,7 +144,7 @@ func (m *Matcher) addScheduler(idx uint) {
 // Start starts the matching process and returns a stream of bloom matches in
 // a given range of blocks. If there are no more matches in the range, the result
 // channel is closed.
-func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession, error) {
+func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) {
 	// Make sure we're not creating concurrent sessions
 	if atomic.SwapUint32(&m.running, 1) == 1 {
 		return nil, errors.New("matcher already running")
@@ -149,6 +156,7 @@ func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession
 		matcher: m,
 		quit:    make(chan struct{}),
 		kill:    make(chan struct{}),
+		ctx:     ctx,
 	}
 	for _, scheduler := range m.schedulers {
 		scheduler.reset()
@@ -184,10 +192,12 @@ func (m *Matcher) Start(begin, end uint64, results chan uint64) (*MatcherSession
 				}
 				// Iterate over all the blocks in the section and return the matching ones
 				for i := first; i <= last; i++ {
-					// Skip the entire byte if no matches are found inside
+					// Skip the entire byte if no matches are found inside (and we're processing an entire byte!)
 					next := res.bitset[(i-sectionStart)/8]
 					if next == 0 {
-						i += 7
+						if i%8 == 0 {
+							i += 7
+						}
 						continue
 					}
 					// Some bit it set, do the actual submatching
@@ -502,25 +512,34 @@ func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
 type MatcherSession struct {
 	matcher *Matcher
 
-	quit chan struct{} // Quit channel to request pipeline termination
+	closer sync.Once     // Sync object to ensure we only ever close once
-	kill chan struct{} // Term channel to signal non-graceful forced shutdown
+	quit   chan struct{} // Quit channel to request pipeline termination
+	kill   chan struct{} // Term channel to signal non-graceful forced shutdown
+
+	ctx context.Context // Context used by the light client to abort filtering
+	err atomic.Value    // Global error to track retrieval failures deep in the chain
+
 	pend sync.WaitGroup
 }
 
 // Close stops the matching process and waits for all subprocesses to terminate
 // before returning. The timeout may be used for graceful shutdown, allowing the
 // currently running retrievals to complete before this time.
-func (s *MatcherSession) Close(timeout time.Duration) {
+func (s *MatcherSession) Close() {
-	// Bail out if the matcher is not running
+	s.closer.Do(func() {
-	select {
+		// Signal termination and wait for all goroutines to tear down
-	case <-s.quit:
+		close(s.quit)
-		return
+		time.AfterFunc(time.Second, func() { close(s.kill) })
-	default:
+		s.pend.Wait()
+	})
+}
+
+// Error returns any failure encountered during the matching session.
+func (s *MatcherSession) Error() error {
+	if err := s.err.Load(); err != nil {
+		return err.(error)
 	}
-	// Signal termination and wait for all goroutines to tear down
+	return nil
-	close(s.quit)
-	time.AfterFunc(timeout, func() { close(s.kill) })
-	s.pend.Wait()
 }
 
 // AllocateRetrieval assigns a bloom bit index to a client process that can either
@@ -618,9 +637,13 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan
 
 		case mux <- request:
 			// Retrieval accepted, something must arrive before we're aborting
-			request <- &Retrieval{Bit: bit, Sections: sections}
+			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
 
 			result := <-request
+			if result.Error != nil {
+				s.err.Store(result.Error)
+				s.Close()
+			}
 			s.DeliverSections(result.Bit, result.Sections, result.Bitsets)
 		}
 	}

core/bloombits/matcher_test.go

@@ -17,6 +17,7 @@
 package bloombits
 
 import (
+	"context"
 	"math/rand"
 	"sync/atomic"
 	"testing"
@@ -30,14 +31,14 @@ const testSectionSize = 4096
 // Tests that wildcard filter rules (nil) can be specified and are handled well.
 func TestMatcherWildcards(t *testing.T) {
 	matcher := NewMatcher(testSectionSize, [][][]byte{
-		[][]byte{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
+		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
-		[][]byte{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
+		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
-		[][]byte{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
+		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
-		[][]byte{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
+		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
-		[][]byte{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
+		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
-		[][]byte{nil, nil},                                               // Wildcard combo, drop rule
+		{nil, nil},                                               // Wildcard combo, drop rule
-		[][]byte{},                                                       // Inited wildcard rule, drop rule
+		{},                                                       // Inited wildcard rule, drop rule
-		nil,                                                              // Proper wildcard rule, drop rule
+		nil,                                                      // Proper wildcard rule, drop rule
 	})
 	if len(matcher.filters) != 3 {
 		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
@@ -55,37 +56,52 @@ func TestMatcherWildcards(t *testing.T) {
 
 // Tests the matcher pipeline on a single continuous workflow without interrupts.
 func TestMatcherContinuous(t *testing.T) {
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, false, 75)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, false, 81)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, false, 36)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
 }
 
 // Tests the matcher pipeline on a constantly interrupted and resumed work pattern
 // with the aim of ensuring data items are requested only once.
 func TestMatcherIntermittent(t *testing.T) {
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 100000, true, 75)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 100000, true, 81)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 10000, true, 36)
+	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
 }
 
 // Tests the matcher pipeline on random input to hopefully catch anomalies.
 func TestMatcherRandom(t *testing.T) {
 	for i := 0; i < 10; i++ {
-		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 10000, 0)
+		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
 	}
 }
 
+// Tests that the matcher can properly find matches if the starting block is
+// shifter from a multiple of 8. This is needed to cover an optimisation with
+// bitset matching https://github.com/EthereumCommonwealth/go-callisto/issues/15309.
+func TestMatcherShifted(t *testing.T) {
+	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
+	// start to get a potential zero byte in the matcher bitset.
+
+	// To keep the second bitset byte zero, the filter must only match for the first
+	// time in block 16, so doing an all-16 bit filter should suffice.
+
+	// To keep the starting block non divisible by 8, block number 9 is the first
+	// that would introduce a shift and not match block 0.
+	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
+}
+
 // Tests that matching on everything doesn't crash (special case internally).
 func TestWildcardMatcher(t *testing.T) {
-	testMatcherBothModes(t, nil, 10000, 0)
+	testMatcherBothModes(t, nil, 0, 10000, 0)
 }
 
 // makeRandomIndexes generates a random filter system, composed on multiple filter
-// criteria, each having one bloom list component for the address and arbitrarilly
+// criteria, each having one bloom list component for the address and arbitrarily
 // many topic bloom list components.
 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
 	res := make([][]bloomIndexes, len(lengths))
@@ -103,9 +119,9 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
 // testMatcherDiffBatches runs the given matches test in single-delivery and also
 // in batches delivery mode, verifying that all kinds of deliveries are handled
 // correctly withn.
-func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32) {
+func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
-	singleton := testMatcher(t, filter, blocks, intermittent, retrievals, 1)
+	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
-	batched := testMatcher(t, filter, blocks, intermittent, retrievals, 16)
+	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
 
 	if singleton != batched {
 		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
@@ -114,9 +130,9 @@ func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, blocks uint64
 
 // testMatcherBothModes runs the given matcher test in both continuous as well as
 // in intermittent mode, verifying that the request counts match each other.
-func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64, retrievals uint32) {
+func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
-	continuous := testMatcher(t, filter, blocks, false, retrievals, 16)
+	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
-	intermittent := testMatcher(t, filter, blocks, true, retrievals, 16)
+	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
 
 	if continuous != intermittent {
 		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
@@ -125,7 +141,7 @@ func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, blocks uint64,
 
 // testMatcher is a generic tester to run the given matcher test and return the
 // number of requests made for cross validation between different modes.
-func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
+func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
 	// Create a new matcher an simulate our explicit random bitsets
 	matcher := NewMatcher(testSectionSize, nil)
 	matcher.filters = filter
@@ -144,14 +160,14 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 	quit := make(chan struct{})
 	matches := make(chan uint64, 16)
 
-	session, err := matcher.Start(0, blocks-1, matches)
+	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
 	if err != nil {
 		t.Fatalf("failed to stat matcher session: %v", err)
 	}
 	startRetrievers(session, quit, &requested, maxReqCount)
 
 	// Iterate over all the blocks and verify that the pipeline produces the correct matches
-	for i := uint64(0); i < blocks; i++ {
+	for i := start; i < blocks; i++ {
 		if expMatch3(filter, i) {
 			match, ok := <-matches
 			if !ok {
@@ -163,13 +179,13 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 			}
 			// If we're testing intermittent mode, abort and restart the pipeline
 			if intermittent {
-				session.Close(time.Second)
+				session.Close()
 				close(quit)
 
 				quit = make(chan struct{})
 				matches = make(chan uint64, 16)
 
-				session, err = matcher.Start(i+1, blocks-1, matches)
+				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
 				if err != nil {
 					t.Fatalf("failed to stat matcher session: %v", err)
 				}
@@ -183,7 +199,7 @@ func testMatcher(t *testing.T, filter [][]bloomIndexes, blocks uint64, intermitt
 		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
 	}
 	// Clean up the session and ensure we match the expected retrieval count
-	session.Close(time.Second)
+	session.Close()
 	close(quit)
 
 	if retrievals != 0 && requested != retrievals {

core/bloombits/scheduler_test.go

@@ -60,7 +60,7 @@ func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
 					req.section,                    // Requested data
 					req.section,                    // Duplicated data (ensure it doesn't double close anything)
 				}, [][]byte{
-					[]byte{},
+					{},
 					new(big.Int).SetUint64(req.section).Bytes(),
 					new(big.Int).SetUint64(req.section).Bytes(),
 				})

core/chain_indexer.go

@@ -36,7 +36,7 @@ import (
 type ChainIndexerBackend interface {
 	// Reset initiates the processing of a new chain segment, potentially terminating
 	// any partially completed operations (in case of a reorg).
-	Reset(section uint64)
+	Reset(section uint64, prevHead common.Hash) error
 
 	// Process crunches through the next header in the chain segment. The caller
 	// will ensure a sequential order of headers.
@@ -46,6 +46,15 @@ type ChainIndexerBackend interface {
 	Commit() error
 }
 
+// ChainIndexerChain interface is used for connecting the indexer to a blockchain
+type ChainIndexerChain interface {
+	// CurrentHeader retrieves the latest locally known header.
+	CurrentHeader() *types.Header
+
+	// SubscribeChainEvent subscribes to new head header notifications.
+	SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription
+}
+
 // ChainIndexer does a post-processing job for equally sized sections of the
 // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
 // connected to the blockchain through the event system by starting a
@@ -100,11 +109,27 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken
 	return c
 }
 
+// AddKnownSectionHead marks a new section head as known/processed if it is newer
+// than the already known best section head
+func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if section < c.storedSections {
+		return
+	}
+	c.setSectionHead(section, shead)
+	c.setValidSections(section + 1)
+}
+
 // Start creates a goroutine to feed chain head events into the indexer for
 // cascading background processing. Children do not need to be started, they
 // are notified about new events by their parents.
-func (c *ChainIndexer) Start(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
+func (c *ChainIndexer) Start(chain ChainIndexerChain) {
-	go c.eventLoop(currentHeader, chainEventer)
+	events := make(chan ChainEvent, 10)
+	sub := chain.SubscribeChainEvent(events)
+
+	go c.eventLoop(chain.CurrentHeader(), events, sub)
 }
 
 // Close tears down all goroutines belonging to the indexer and returns any error
@@ -147,12 +172,10 @@ func (c *ChainIndexer) Close() error {
 // eventLoop is a secondary - optional - event loop of the indexer which is only
 // started for the outermost indexer to push chain head events into a processing
 // queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(ch chan<- ChainEvent) event.Subscription) {
+func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) {
 	// Mark the chain indexer as active, requiring an additional teardown
 	atomic.StoreUint32(&c.active, 1)
 
-	events := make(chan ChainEvent, 10)
-	sub := chainEventer(events)
 	defer sub.Unsubscribe()
 
 	// Fire the initial new head event to start any outstanding processing
@@ -178,7 +201,11 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, chainEventer func(
 			}
 			header := ev.Block.Header()
 			if header.ParentHash != prevHash {
-				c.newHead(FindCommonAncestor(c.chainDb, prevHeader, header).Number.Uint64(), true)
+				// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
+				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
+				if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
+					c.newHead(h.Number.Uint64(), true)
+				}
 			}
 			c.newHead(header.Number.Uint64(), false)
 
@@ -203,7 +230,7 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) {
 		if changed < c.storedSections {
 			c.setValidSections(changed)
 		}
-		// Update the new head number to te finalized section end and notify children
+		// Update the new head number to the finalized section end and notify children
 		head = changed * c.sectionSize
 
 		if head < c.cascadedHead {
@@ -236,6 +263,7 @@ func (c *ChainIndexer) updateLoop() {
 		updating bool
 		updated  time.Time
 	)
+
 	for {
 		select {
 		case errc := <-c.quit:
@@ -259,7 +287,7 @@ func (c *ChainIndexer) updateLoop() {
 				section := c.storedSections
 				var oldHead common.Hash
 				if section > 0 {
-					oldHead = c.sectionHead(section - 1)
+					oldHead = c.SectionHead(section - 1)
 				}
 				// Process the newly defined section in the background
 				c.lock.Unlock()
@@ -270,7 +298,7 @@ func (c *ChainIndexer) updateLoop() {
 				c.lock.Lock()
 
 				// If processing succeeded and no reorgs occcurred, mark the section completed
-				if err == nil && oldHead == c.sectionHead(section-1) {
+				if err == nil && oldHead == c.SectionHead(section-1) {
 					c.setSectionHead(section, newHead)
 					c.setValidSections(section + 1)
 					if c.storedSections == c.knownSections && updating {
@@ -311,7 +339,11 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com
 	c.log.Trace("Processing new chain section", "section", section)
 
 	// Reset and partial processing
-	c.backend.Reset(section)
+
+	if err := c.backend.Reset(section, lastHead); err != nil {
+		c.setValidSections(0)
+		return common.Hash{}, err
+	}
 
 	for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
 		hash := GetCanonicalHash(c.chainDb, number)
@@ -341,7 +373,7 @@ func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	return c.storedSections, c.storedSections*c.sectionSize - 1, c.sectionHead(c.storedSections - 1)
+	return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
 }
 
 // AddChildIndexer adds a child ChainIndexer that can use the output of this one
@@ -381,9 +413,9 @@ func (c *ChainIndexer) setValidSections(sections uint64) {
 	c.storedSections = sections // needed if new > old
 }
 
-// sectionHead retrieves the last block hash of a processed section from the
+// SectionHead retrieves the last block hash of a processed section from the
 // index database.
-func (c *ChainIndexer) sectionHead(section uint64) common.Hash {
+func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
 	var data [8]byte
 	binary.BigEndian.PutUint64(data[:], section)
 

core/chain_indexer_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/core/types"
 	"github.com/EthereumCommonwealth/go-callisto/ethdb"
 )
@@ -208,9 +209,10 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
 	return b.stored * b.indexer.sectionSize
 }
 
-func (b *testChainIndexBackend) Reset(section uint64) {
+func (b *testChainIndexBackend) Reset(section uint64, prevHead common.Hash) error {
 	b.section = section
 	b.headerCnt = 0
+	return nil
 }
 
 func (b *testChainIndexBackend) Process(header *types.Header) {

core/chain_makers.go

@@ -235,7 +235,7 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockChain, error) {
 	db, _ := ethdb.NewMemDatabase()
 	genesis := gspec.MustCommit(db)
 
-	blockchain, _ := NewBlockChain(db, params.AllProtocolChanges, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, params.AllEthashProtocolChanges, ethash.NewFaker(), vm.Config{})
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil

core/database_util.go

@@ -74,9 +74,9 @@ var (
 	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
 )
 
-// txLookupEntry is a positional metadata to help looking up the data content of
+// TxLookupEntry is a positional metadata to help looking up the data content of
 // a transaction or receipt given only its hash.
-type txLookupEntry struct {
+type TxLookupEntry struct {
 	BlockHash  common.Hash
 	BlockIndex uint64
 	Index      uint64
@@ -260,7 +260,7 @@ func GetTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64,
 		return common.Hash{}, 0, 0
 	}
 	// Parse and return the contents of the lookup entry
-	var entry txLookupEntry
+	var entry TxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
 		log.Error("Invalid lookup entry RLP", "hash", hash, "err", err)
 		return common.Hash{}, 0, 0
@@ -296,7 +296,7 @@ func GetTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, co
 	if len(data) == 0 {
 		return nil, common.Hash{}, 0, 0
 	}
-	var entry txLookupEntry
+	var entry TxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
 		return nil, common.Hash{}, 0, 0
 	}
@@ -332,14 +332,13 @@ func GetReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Has
 
 // GetBloomBits retrieves the compressed bloom bit vector belonging to the given
 // section and bit index from the.
-func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) []byte {
+func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) {
 	key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...)
 
 	binary.BigEndian.PutUint16(key[1:], uint16(bit))
 	binary.BigEndian.PutUint64(key[3:], section)
 
-	bits, _ := db.Get(key)
+	return db.Get(key)
-	return bits
 }
 
 // WriteCanonicalHash stores the canonical hash for the given block number.
@@ -465,7 +464,7 @@ func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receip
 func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error {
 	// Iterate over each transaction and encode its metadata
 	for i, tx := range block.Transactions() {
-		entry := txLookupEntry{
+		entry := TxLookupEntry{
 			BlockHash:  block.Hash(),
 			BlockIndex: block.NumberU64(),
 			Index:      uint64(i),

core/genesis.go

@@ -151,7 +151,7 @@ func (e *GenesisMismatchError) Error() string {
 // The returned chain configuration is never nil.
 func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) {
 	if genesis != nil && genesis.Config == nil {
-		return params.AllProtocolChanges, common.Hash{}, errGenesisNoConfig
+		return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
 
 	// Just commit the new block if there is no stored genesis block.
@@ -216,7 +216,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 	case ghash == params.TestnetGenesisHash:
 		return params.TestnetChainConfig
 	default:
-		return params.AllProtocolChanges
+		return params.AllEthashProtocolChanges
 	}
 }
 
@@ -285,7 +285,7 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
 	}
 	config := g.Config
 	if config == nil {
-		config = params.AllProtocolChanges
+		config = params.AllEthashProtocolChanges
 	}
 	return block, WriteChainConfig(db, block.Hash(), config)
 }
@@ -342,14 +342,30 @@ func DefaultRinkebyGenesisBlock() *Genesis {
 	}
 }
 
-// DevGenesisBlock returns the 'geth --dev' genesis block.
+// DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must
-func DevGenesisBlock() *Genesis {
+// be seeded with the
+func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis {
+	// Override the default period to the user requested one
+	config := *params.AllCliqueProtocolChanges
+	config.Clique.Period = period
+
+	// Assemble and return the genesis with the precompiles and faucet pre-funded
 	return &Genesis{
-		Config:     params.AllProtocolChanges,
+		Config:     &config,
-		Nonce:      42,
+		ExtraData:  append(append(make([]byte, 32), faucet[:]...), make([]byte, 65)...),
-		GasLimit:   4712388,
+		GasLimit:   6283185,
-		Difficulty: big.NewInt(131072),
+		Difficulty: big.NewInt(1),
-		Alloc:      decodePrealloc(devAllocData),
+		Alloc: map[common.Address]GenesisAccount{
+			common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover
+			common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256
+			common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD
+			common.BytesToAddress([]byte{4}): {Balance: big.NewInt(1)}, // Identity
+			common.BytesToAddress([]byte{5}): {Balance: big.NewInt(1)}, // ModExp
+			common.BytesToAddress([]byte{6}): {Balance: big.NewInt(1)}, // ECAdd
+			common.BytesToAddress([]byte{7}): {Balance: big.NewInt(1)}, // ECScalarMul
+			common.BytesToAddress([]byte{8}): {Balance: big.NewInt(1)}, // ECPairing
+			faucet: {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))},
+		},
 	}
 }
 

core/genesis_test.go

@@ -65,7 +65,7 @@ func TestSetupGenesis(t *testing.T) {
 				return SetupGenesisBlock(db, new(Genesis))
 			},
 			wantErr:    errGenesisNoConfig,
-			wantConfig: params.AllProtocolChanges,
+			wantConfig: params.AllEthashProtocolChanges,
 		},
 		{
 			name: "no block in DB, genesis == nil",

core/state/statedb.go

@@ -453,7 +453,7 @@ func (self *StateDB) Copy() *StateDB {
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
 		db:                self.db,
-		trie:              self.trie,
+		trie:              self.db.CopyTrie(self.trie),
 		stateObjects:      make(map[common.Address]*stateObject, len(self.stateObjectsDirty)),
 		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
 		refund:            new(big.Int).Set(self.refund),

core/state/statedb_test.go

@@ -117,6 +117,57 @@ func TestIntermediateLeaks(t *testing.T) {
 	}
 }
 
+// TestCopy tests that copying a statedb object indeed makes the original and
+// the copy independent of each other. This test is a regression test against
+// https://github.com/EthereumCommonwealth/go-callisto/pull/15549.
+func TestCopy(t *testing.T) {
+	// Create a random state test to copy and modify "independently"
+	mem, _ := ethdb.NewMemDatabase()
+	orig, _ := New(common.Hash{}, NewDatabase(mem))
+
+	for i := byte(0); i < 255; i++ {
+		obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+		obj.AddBalance(big.NewInt(int64(i)))
+		orig.updateStateObject(obj)
+	}
+	orig.Finalise(false)
+
+	// Copy the state, modify both in-memory
+	copy := orig.Copy()
+
+	for i := byte(0); i < 255; i++ {
+		origObj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+		copyObj := copy.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+
+		origObj.AddBalance(big.NewInt(2 * int64(i)))
+		copyObj.AddBalance(big.NewInt(3 * int64(i)))
+
+		orig.updateStateObject(origObj)
+		copy.updateStateObject(copyObj)
+	}
+	// Finalise the changes on both concurrently
+	done := make(chan struct{})
+	go func() {
+		orig.Finalise(true)
+		close(done)
+	}()
+	copy.Finalise(true)
+	<-done
+
+	// Verify that the two states have been updated independently
+	for i := byte(0); i < 255; i++ {
+		origObj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+		copyObj := copy.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+
+		if want := big.NewInt(3 * int64(i)); origObj.Balance().Cmp(want) != 0 {
+			t.Errorf("orig obj %d: balance mismatch: have %v, want %v", i, origObj.Balance(), want)
+		}
+		if want := big.NewInt(4 * int64(i)); copyObj.Balance().Cmp(want) != 0 {
+			t.Errorf("copy obj %d: balance mismatch: have %v, want %v", i, copyObj.Balance(), want)
+		}
+	}
+}
+
 func TestSnapshotRandom(t *testing.T) {
 	config := &quick.Config{MaxCount: 1000}
 	err := quick.Check((*snapshotTest).run, config)

core/tx_journal.go

@@ -31,6 +31,15 @@ import (
 // into the journal, but no such file is currently open.
 var errNoActiveJournal = errors.New("no active journal")
 
+// devNull is a WriteCloser that just discards anything written into it. Its
+// goal is to allow the transaction journal to write into a fake journal when
+// loading transactions on startup without printing warnings due to no file
+// being readt for write.
+type devNull struct{}
+
+func (*devNull) Write(p []byte) (n int, err error) { return len(p), nil }
+func (*devNull) Close() error                      { return nil }
+
 // txJournal is a rotating log of transactions with the aim of storing locally
 // created transactions to allow non-executed ones to survive node restarts.
 type txJournal struct {
@@ -59,6 +68,10 @@ func (journal *txJournal) load(add func(*types.Transaction) error) error {
 	}
 	defer input.Close()
 
+	// Temporarily discard any journal additions (don't double add on load)
+	journal.writer = new(devNull)
+	defer func() { journal.writer = nil }()
+
 	// Inject all transactions from the journal into the pool
 	stream := rlp.NewStream(input, 0)
 	total, dropped := 0, 0

core/tx_list.go

@@ -254,7 +254,10 @@ func (l *txList) Add(tx *types.Transaction, priceBump uint64) (bool, *types.Tran
 	old := l.txs.Get(tx.Nonce())
 	if old != nil {
 		threshold := new(big.Int).Div(new(big.Int).Mul(old.GasPrice(), big.NewInt(100+int64(priceBump))), big.NewInt(100))
-		if threshold.Cmp(tx.GasPrice()) >= 0 {
+		// Have to ensure that the new gas price is higher than the old gas
+		// price as well as checking the percentage threshold to ensure that
+		// this is accurate for low (Wei-level) gas price replacements
+		if old.GasPrice().Cmp(tx.GasPrice()) >= 0 || threshold.Cmp(tx.GasPrice()) > 0 {
 			return false, nil
 		}
 	}

core/tx_pool.go

@@ -103,6 +103,16 @@ var (
 	underpricedTxCounter = metrics.NewCounter("txpool/underpriced")
 )
 
+// TxStatus is the current status of a transaction as seen py the pool.
+type TxStatus uint
+
+const (
+	TxStatusUnknown TxStatus = iota
+	TxStatusQueued
+	TxStatusPending
+	TxStatusIncluded
+)
+
 // blockChain provides the state of blockchain and current gas limit to do
 // some pre checks in tx pool and event subscribers.
 type blockChain interface {
@@ -640,6 +650,10 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		pool.journalTx(from, tx)
 
 		log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To())
+
+		// We've directly injected a replacement transaction, notify subsystems
+		go pool.txFeed.Send(TxPreEvent{tx})
+
 		return old != nil, nil
 	}
 	// New transaction isn't replacing a pending one, push into queue
@@ -729,6 +743,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T
 	// Set the potentially new pending nonce and notify any subsystems of the new tx
 	pool.beats[addr] = time.Now()
 	pool.pendingState.SetNonce(addr, tx.Nonce()+1)
+
 	go pool.txFeed.Send(TxPreEvent{tx})
 }
 
@@ -749,14 +764,14 @@ func (pool *TxPool) AddRemote(tx *types.Transaction) error {
 // AddLocals enqueues a batch of transactions into the pool if they are valid,
 // marking the senders as a local ones in the mean time, ensuring they go around
 // the local pricing constraints.
-func (pool *TxPool) AddLocals(txs []*types.Transaction) error {
+func (pool *TxPool) AddLocals(txs []*types.Transaction) []error {
 	return pool.addTxs(txs, !pool.config.NoLocals)
 }
 
 // AddRemotes enqueues a batch of transactions into the pool if they are valid.
 // If the senders are not among the locally tracked ones, full pricing constraints
 // will apply.
-func (pool *TxPool) AddRemotes(txs []*types.Transaction) error {
+func (pool *TxPool) AddRemotes(txs []*types.Transaction) []error {
 	return pool.addTxs(txs, false)
 }
 
@@ -779,7 +794,7 @@ func (pool *TxPool) addTx(tx *types.Transaction, local bool) error {
 }
 
 // addTxs attempts to queue a batch of transactions if they are valid.
-func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error {
+func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 	pool.mu.Lock()
 	defer pool.mu.Unlock()
 
@@ -788,11 +803,14 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error {
 
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
-func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error {
+func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
 	// Add the batch of transaction, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
-	for _, tx := range txs {
+	errs := make([]error, len(txs))
-		if replace, err := pool.add(tx, local); err == nil {
+
+	for i, tx := range txs {
+		var replace bool
+		if replace, errs[i] = pool.add(tx, local); errs[i] == nil {
 			if !replace {
 				from, _ := types.Sender(pool.signer, tx) // already validated
 				dirty[from] = struct{}{}
@@ -802,12 +820,32 @@ func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) error {
 	// Only reprocess the internal state if something was actually added
 	if len(dirty) > 0 {
 		addrs := make([]common.Address, 0, len(dirty))
-		for addr, _ := range dirty {
+		for addr := range dirty {
 			addrs = append(addrs, addr)
 		}
 		pool.promoteExecutables(addrs)
 	}
-	return nil
+	return errs
+}
+
+// Status returns the status (unknown/pending/queued) of a batch of transactions
+// identified by their hashes.
+func (pool *TxPool) Status(hashes []common.Hash) []TxStatus {
+	pool.mu.RLock()
+	defer pool.mu.RUnlock()
+
+	status := make([]TxStatus, len(hashes))
+	for i, hash := range hashes {
+		if tx := pool.all[hash]; tx != nil {
+			from, _ := types.Sender(pool.signer, tx) // already validated
+			if pool.pending[from].txs.items[tx.Nonce()] != nil {
+				status[i] = TxStatusPending
+			} else {
+				status[i] = TxStatusQueued
+			}
+		}
+	}
+	return status
 }
 
 // Get returns a transaction if it is contained in the pool
@@ -869,7 +907,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 	// Gather all the accounts potentially needing updates
 	if accounts == nil {
 		accounts = make([]common.Address, 0, len(pool.queue))
-		for addr, _ := range pool.queue {
+		for addr := range pool.queue {
 			accounts = append(accounts, addr)
 		}
 	}