Merge remote-tracking branch 'upstream/master'

.github/CODEOWNERS

@@ -6,6 +6,7 @@ accounts/abi                    @gballet
 consensus                       @karalabe
 core/                           @karalabe @holiman
 eth/                            @karalabe
+graphql/                        @gballet
 les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe

.gitignore

@@ -42,6 +42,7 @@ profile.cov
 /dashboard/assets/node_modules
 /dashboard/assets/stats.json
 /dashboard/assets/bundle.js
+/dashboard/assets/bundle.js.map
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log

.travis.yml

@@ -14,7 +14,6 @@ matrix:
       - go run build/ci.go install
       - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
@@ -26,8 +25,20 @@ matrix:
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
+    # These are the latest Go versions.
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.12.x
+      script:
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
     - os: osx
-      go: 1.11.x
+      go: 1.12.x
       script:
         - echo "Increase the maximum number of open file descriptors on macOS"
         - NOFILE=20480
@@ -44,7 +55,7 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - lint
       git:
@@ -53,10 +64,10 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - if: type = push
+    - if: repo = ethereum/go-ethereum AND type = push
       os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - ubuntu-ppa
       git:
@@ -75,11 +86,11 @@ matrix:
         - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
-    - if: type = push
+    - if: repo = ethereum/go-ethereum AND type = push
       os: linux
       dist: trusty
       sudo: required
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-linux
       git:
@@ -109,12 +120,12 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - if: type = push
+    - if: repo = ethereum/go-ethereum AND type = push
       os: linux
       dist: trusty
       services:
         - docker
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-linux-mips
       git:
@@ -137,7 +148,7 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - if: type = push
+    - if: repo = ethereum/go-ethereum AND type = push
       os: linux
       dist: trusty
       addons:
@@ -159,25 +170,24 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.5.linux-amd64.tar.gz | tar -xz
+        - curl https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip -o android-ndk-r17b.zip
+        - curl https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip -o android-ndk-r19b.zip
-        - unzip -q android-ndk-r17b.zip && rm android-ndk-r17b.zip
+        - unzip -q android-ndk-r19b.zip && rm android-ndk-r19b.zip
-        - mv android-ndk-r17b $HOME
+        - mv android-ndk-r19b $ANDROID_HOME/ndk-bundle
-        - export ANDROID_NDK=$HOME/android-ndk-r17b
 
         - mkdir -p $GOPATH/src/github.com/ethereum
-        - ln -s `pwd` $GOPATH/src/github.com/ethereum
+        - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - if: type = push
+    - if: repo = ethereum/go-ethereum AND type = push
       os: osx
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-osx
         - azure-ios
@@ -204,13 +214,22 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - if: type = cron
+    - if: repo = ethereum/go-ethereum AND type = cron
       os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-purge
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
+
+    - name: Race Detector for Swarm
+      if: repo = ethersphere/go-ethereum
+      os: linux
+      dist: trusty
+      go: 1.12.x
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm...

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.11-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.11-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

README.md

@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.9 or later) and a C compiler.
+Building geth requires both a Go (version 1.10 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 

accounts/abi/abi.go

@@ -136,7 +136,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
 	if len(sigdata) < 4 {
-		return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
+		return nil, fmt.Errorf("data too short (%d bytes) for abi method lookup", len(sigdata))
 	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {

accounts/abi/bind/backends/simulated.go

@@ -66,7 +66,7 @@ type SimulatedBackend struct {
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
 func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	database := ethdb.NewMemDatabase()
+	database := rawdb.NewMemoryDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
 	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
@@ -168,18 +168,18 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 // blockchain. The isPending return value indicates whether the transaction has been
 // mined yet. Note that the transaction may not be part of the canonical chain even if
 // it's not pending.
-func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) {
+func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (*types.Transaction, bool, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
 
-	tx = b.pendingBlock.Transaction(txHash)
+	tx := b.pendingBlock.Transaction(txHash)
 	if tx != nil {
 		return tx, true, nil
 	}
-
 	tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
 	if tx != nil {
 		return tx, false, nil
 	}
-
 	return nil, false, ethereum.NotFound
 }
 

accounts/abi/bind/backends/simulated_test.go

@@ -1,3 +1,19 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package backends_test
 
 import (

accounts/abi/bind/base_test.go

@@ -1,3 +1,19 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package bind_test
 
 import (

accounts/abi/bind/bind_test.go

@@ -698,10 +698,18 @@ var bindTests = []struct {
 				function raiseDynamicEvent(string str, bytes blob) {
 					DynamicEvent(str, blob, str, blob);
 				}
+
+				event FixedBytesEvent (
+					bytes24 indexed IndexedBytes,
+					bytes24 NonIndexedBytes
+				);
+				function raiseFixedBytesEvent(bytes24 blob) {
+					FixedBytesEvent(blob, blob);
+				}
 			}
 		`,
-		`6060604052341561000f57600080fd5b61042c8061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063528300ff1461005c578063630c31e2146100fc578063c7d116dd14610156575b600080fd5b341561006757600080fd5b6100fa600480803590602001908201803590602001908080601f0160208091040260200160405190810160405280939291908181526020018383808284378201915050505050509190803590602001908201803590602001908080601f01602080910402602001604051908101604052809392919081815260200183838082843782019150505050505091905050610194565b005b341561010757600080fd5b610154600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091908035600019169060200190919080351515906020019091908035906020019091905050610367565b005b341561016157600080fd5b610192600480803590602001909190803560010b90602001909190803563ffffffff169060200190919050506103c3565b005b806040518082805190602001908083835b6020831015156101ca57805182526020820191506020810190506020830392506101a5565b6001836020036101000a0380198251168184511680821785525050505050509050019150506040518091039020826040518082805190602001908083835b60208310151561022d5780518252602082019150602081019050602083039250610208565b6001836020036101000a03801982511681845116808217855250505050505090500191505060405180910390207f3281fd4f5e152dd3385df49104a3f633706e21c9e80672e88d3bcddf33101f008484604051808060200180602001838103835285818151815260200191508051906020019080838360005b838110156102c15780820151818401526020810190506102a6565b50505050905090810190601f1680156102ee5780820380516001836020036101000a031916815260200191505b50838103825284818151815260200191508051906020019080838360005b8381101561032757808201518184015260208101905061030c565b50505050905090810190601f1680156103545780820380516001836020036101000a031916815260200191505b5094505050505060405180910390a35050565b81151583600019168573ffffffffffffffffffffffffffffffffffffffff167f1f097de4289df643bd9c11011cc61367aa12983405c021056e706eb5ba1250c8846040518082815260200191505060405180910390a450505050565b8063ffffffff168260010b847f3ca7f3a77e5e6e15e781850bc82e32adfa378a2a609370db24b4d0fae10da2c960405160405180910390a45050505600a165627a7a72305820d1f8a8bbddbc5bb29f285891d6ae1eef8420c52afdc05e1573f6114d8e1714710029`,
+		`608060405234801561001057600080fd5b5061043f806100206000396000f3006080604052600436106100615763ffffffff7c0100000000000000000000000000000000000000000000000000000000600035041663528300ff8114610066578063630c31e2146100ff5780636cc6b94014610138578063c7d116dd1461015b575b600080fd5b34801561007257600080fd5b506040805160206004803580820135601f81018490048402850184019095528484526100fd94369492936024939284019190819084018382808284375050604080516020601f89358b018035918201839004830284018301909452808352979a9998810197919650918201945092508291508401838280828437509497506101829650505050505050565b005b34801561010b57600080fd5b506100fd73ffffffffffffffffffffffffffffffffffffffff60043516602435604435151560643561033c565b34801561014457600080fd5b506100fd67ffffffffffffffff1960043516610394565b34801561016757600080fd5b506100fd60043560243560010b63ffffffff604435166103d6565b806040518082805190602001908083835b602083106101b25780518252601f199092019160209182019101610193565b51815160209384036101000a6000190180199092169116179052604051919093018190038120875190955087945090928392508401908083835b6020831061020b5780518252601f1990920191602091820191016101ec565b6001836020036101000a03801982511681845116808217855250505050505090500191505060405180910390207f3281fd4f5e152dd3385df49104a3f633706e21c9e80672e88d3bcddf33101f008484604051808060200180602001838103835285818151815260200191508051906020019080838360005b8381101561029c578181015183820152602001610284565b50505050905090810190601f1680156102c95780820380516001836020036101000a031916815260200191505b50838103825284518152845160209182019186019080838360005b838110156102fc5781810151838201526020016102e4565b50505050905090810190601f1680156103295780820380516001836020036101000a031916815260200191505b5094505050505060405180910390a35050565b60408051828152905183151591859173ffffffffffffffffffffffffffffffffffffffff8816917f1f097de4289df643bd9c11011cc61367aa12983405c021056e706eb5ba1250c8919081900360200190a450505050565b6040805167ffffffffffffffff19831680825291517fcdc4c1b1aed5524ffb4198d7a5839a34712baef5fa06884fac7559f4a5854e0a9181900360200190a250565b8063ffffffff168260010b847f3ca7f3a77e5e6e15e781850bc82e32adfa378a2a609370db24b4d0fae10da2c960405160405180910390a45050505600a165627a7a72305820468b5843bf653145bd924b323c64ef035d3dd922c170644b44d61aa666ea6eee0029`,
-		`[{"constant":false,"inputs":[{"name":"str","type":"string"},{"name":"blob","type":"bytes"}],"name":"raiseDynamicEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"},{"name":"id","type":"bytes32"},{"name":"flag","type":"bool"},{"name":"value","type":"uint256"}],"name":"raiseSimpleEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"number","type":"uint256"},{"name":"short","type":"int16"},{"name":"long","type":"uint32"}],"name":"raiseNodataEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Addr","type":"address"},{"indexed":true,"name":"Id","type":"bytes32"},{"indexed":true,"name":"Flag","type":"bool"},{"indexed":false,"name":"Value","type":"uint256"}],"name":"SimpleEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Number","type":"uint256"},{"indexed":true,"name":"Short","type":"int16"},{"indexed":true,"name":"Long","type":"uint32"}],"name":"NodataEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedString","type":"string"},{"indexed":true,"name":"IndexedBytes","type":"bytes"},{"indexed":false,"name":"NonIndexedString","type":"string"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes"}],"name":"DynamicEvent","type":"event"}]`,
+		`[{"constant":false,"inputs":[{"name":"str","type":"string"},{"name":"blob","type":"bytes"}],"name":"raiseDynamicEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"},{"name":"id","type":"bytes32"},{"name":"flag","type":"bool"},{"name":"value","type":"uint256"}],"name":"raiseSimpleEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"blob","type":"bytes24"}],"name":"raiseFixedBytesEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"number","type":"uint256"},{"name":"short","type":"int16"},{"name":"long","type":"uint32"}],"name":"raiseNodataEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Addr","type":"address"},{"indexed":true,"name":"Id","type":"bytes32"},{"indexed":true,"name":"Flag","type":"bool"},{"indexed":false,"name":"Value","type":"uint256"}],"name":"SimpleEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Number","type":"uint256"},{"indexed":true,"name":"Short","type":"int16"},{"indexed":true,"name":"Long","type":"uint32"}],"name":"NodataEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedString","type":"string"},{"indexed":true,"name":"IndexedBytes","type":"bytes"},{"indexed":false,"name":"NonIndexedString","type":"string"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes"}],"name":"DynamicEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedBytes","type":"bytes24"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes24"}],"name":"FixedBytesEvent","type":"event"}]`,
 		`
 			"math/big"
 			"time"
@@ -812,6 +820,33 @@ var bindTests = []struct {
 			if err = dit.Error(); err != nil {
 				t.Fatalf("dynamic event iteration failed: %v", err)
 			}
+			// Test raising and filtering for events with fixed bytes components
+			var fblob [24]byte
+			copy(fblob[:], []byte("Fixed Bytes"))
+
+			if _, err := eventer.RaiseFixedBytesEvent(auth, fblob); err != nil {
+				t.Fatalf("failed to raise fixed bytes event: %v", err)
+			}
+			sim.Commit()
+
+			fit, err := eventer.FilterFixedBytesEvent(nil, [][24]byte{fblob})
+			if err != nil {
+				t.Fatalf("failed to filter for fixed bytes events: %v", err)
+			}
+			defer fit.Close()
+
+			if !fit.Next() {
+				t.Fatalf("fixed bytes log not found: %v", fit.Error())
+			}
+			if fit.Event.NonIndexedBytes != fblob || fit.Event.IndexedBytes != fblob {
+				t.Errorf("fixed bytes log content mismatch: have %v, want {'%x', '%x'}", fit.Event, fblob, fblob)
+			}
+			if fit.Next() {
+				t.Errorf("unexpected fixed bytes event found: %+v", fit.Event)
+			}
+			if err = fit.Error(); err != nil {
+				t.Fatalf("fixed bytes event iteration failed: %v", err)
+			}
 			// Test subscribing to an event and raising it afterwards
 			ch := make(chan *EventerSimpleEvent, 16)
 			sub, err := eventer.WatchSimpleEvent(nil, ch, nil, nil, nil)

accounts/abi/bind/topics.go

@@ -83,8 +83,10 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
 				val := reflect.ValueOf(rule)
 
 				switch {
+
+				// static byte array
 				case val.Kind() == reflect.Array && reflect.TypeOf(rule).Elem().Kind() == reflect.Uint8:
-					reflect.Copy(reflect.ValueOf(topic[common.HashLength-val.Len():]), val)
+					reflect.Copy(reflect.ValueOf(topic[:val.Len()]), val)
 
 				default:
 					return nil, fmt.Errorf("unsupported indexed type: %T", rule)
@@ -175,8 +177,10 @@ func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) er
 			default:
 				// Ran out of custom types, try the crazies
 				switch {
+
+				// static byte array
 				case arg.Type.T == abi.FixedBytesTy:
-					reflect.Copy(field, reflect.ValueOf(topics[0][common.HashLength-arg.Type.Size:]))
+					reflect.Copy(field, reflect.ValueOf(topics[0][:arg.Type.Size]))
 
 				default:
 					return fmt.Errorf("unsupported indexed type: %v", arg.Type)

accounts/abi/bind/topics_test.go

@@ -0,0 +1,103 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bind
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func TestMakeTopics(t *testing.T) {
+	type args struct {
+		query [][]interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    [][]common.Hash
+		wantErr bool
+	}{
+		{
+			"support fixed byte types, right padded to 32 bytes",
+			args{[][]interface{}{{[5]byte{1, 2, 3, 4, 5}}}},
+			[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := makeTopics(tt.args.query...)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("makeTopics() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("makeTopics() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseTopics(t *testing.T) {
+	type bytesStruct struct {
+		StaticBytes [5]byte
+	}
+	bytesType, _ := abi.NewType("bytes5", nil)
+	type args struct {
+		createObj func() interface{}
+		resultObj func() interface{}
+		fields    abi.Arguments
+		topics    []common.Hash
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "support fixed byte types, right padded to 32 bytes",
+			args: args{
+				createObj: func() interface{} { return &bytesStruct{} },
+				resultObj: func() interface{} { return &bytesStruct{StaticBytes: [5]byte{1, 2, 3, 4, 5}} },
+				fields: abi.Arguments{abi.Argument{
+					Name:    "staticBytes",
+					Type:    bytesType,
+					Indexed: true,
+				}},
+				topics: []common.Hash{
+					{1, 2, 3, 4, 5},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			createObj := tt.args.createObj()
+			if err := parseTopics(createObj, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr {
+				t.Errorf("parseTopics() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			resultObj := tt.args.resultObj()
+			if !reflect.DeepEqual(createObj, resultObj) {
+				t.Errorf("parseTopics() = %v, want %v", createObj, resultObj)
+			}
+		})
+	}
+}

accounts/accounts.go

@@ -104,7 +104,7 @@ type Wallet interface {
 	// a password to decrypt the account, or a PIN code o verify the transaction),
 	// an AuthNeededError instance will be returned, containing infos for the user
 	// about which fields or actions are needed. The user may retry by providing
-	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the needed details via SignDataWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
 	SignData(account Account, mimeType string, data []byte) ([]byte, error)
 
@@ -114,7 +114,7 @@ type Wallet interface {
 	// should never echo the mimetype or return the mimetype in the error-response
 	SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error)
 
-	// Signtext requests the wallet to sign the hash of a given piece of data, prefixed
+	// SignText requests the wallet to sign the hash of a given piece of data, prefixed
 	// by the Ethereum prefix scheme
 	// It looks up the account specified either solely via its address contained within,
 	// or optionally with the aid of any location metadata from the embedded URL field.

accounts/keystore/keystore.go

@@ -137,8 +137,10 @@ func (ks *KeyStore) refreshWallets() {
 	accs := ks.cache.accounts()
 
 	// Transform the current list of wallets into the new one
-	wallets := make([]accounts.Wallet, 0, len(accs))
+	var (
-	events := []accounts.WalletEvent{}
+		wallets = make([]accounts.Wallet, 0, len(accs))
+		events  []accounts.WalletEvent
+	)
 
 	for _, account := range accs {
 		// Drop wallets while they were in front of the next account

accounts/keystore/wallet.go

@@ -111,7 +111,7 @@ func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte
 	return w.signHash(account, accounts.TextHash(text))
 }
 
-// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
+// SignTextWithPassphrase implements accounts.Wallet, attempting to sign the
 // given hash with the given account using passphrase as extra authentication.
 func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
 	// Make sure the requested account is contained within

accounts/usbwallet/hub.go

@@ -150,8 +150,10 @@ func (hub *Hub) refreshWallets() {
 	// Transform the current list of wallets into the new one
 	hub.stateLock.Lock()
 
-	wallets := make([]accounts.Wallet, 0, len(devices))
+	var (
-	events := []accounts.WalletEvent{}
+		wallets = make([]accounts.Wallet, 0, len(devices))
+		events  []accounts.WalletEvent
+	)
 
 	for _, device := range devices {
 		url := accounts.URL{Scheme: hub.scheme, Path: device.Path}

accounts/usbwallet/wallet.go

@@ -274,9 +274,7 @@ func (w *wallet) close() error {
 	w.device = nil
 
 	w.accounts, w.paths = nil, nil
-	w.driver.Close()
+	return w.driver.Close()
-
-	return nil
 }
 
 // Accounts implements accounts.Wallet, returning the list of accounts pinned to

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.5.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://dl.google.com/go/go1.12.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.12.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci-notes.md

@@ -23,18 +23,18 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.10, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.10 and Debian packaging tools:
+Add the gophers PPA and install Go 1.11 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
+    $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -80,6 +80,7 @@ var (
 		executablePath("puppeth"),
 		executablePath("rlpdump"),
 		executablePath("wnode"),
+		executablePath("clef"),
 	}
 
 	// Files that end up in the swarm*.zip archive.
@@ -118,6 +119,10 @@ var (
 			BinaryName:  "wnode",
 			Description: "Ethereum Whisper diagnostic tool",
 		},
+		{
+			BinaryName:  "clef",
+			Description: "Ethereum account management tool.",
+		},
 	}
 
 	// A debian package is created for all executables listed here.
@@ -156,7 +161,7 @@ var (
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
 	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
 	// Note: artful is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
+	debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -800,12 +805,8 @@ func doAndroidArchive(cmdline []string) {
 	if os.Getenv("ANDROID_HOME") == "" {
 		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
 	}
-	if os.Getenv("ANDROID_NDK") == "" {
-		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
-	}
 	// Build the Android archive and Maven resources
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
-	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {

build/deb/ethereum-swarm/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.10
+Build-Depends: debhelper (>= 8.0.0), golang-1.11
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb/ethereum-swarm/deb.rules

@@ -4,8 +4,11 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/deb/ethereum/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.10
+Build-Depends: debhelper (>= 8.0.0), golang-1.11
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb/ethereum/deb.rules

@@ -4,8 +4,11 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/travis_keepalive.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# travis_keepalive runs the given command and preserves its return value,
+# while it forks a child process what periodically produces a log line,
+# so that Travis won't abort the build after 10 minutes.
+
+# Why?
+# `t.Log()` in Go holds the buffer until the test does not pass or fail,
+# and `-race` can increase the execution time by 2-20x.
+
+set -euo pipefail
+
+readonly KEEPALIVE_INTERVAL=300 # seconds => 5m
+
+main() {
+  keepalive
+  $@
+}
+
+# Keepalive produces a log line in each KEEPALIVE_INTERVAL.
+keepalive() {
+  local child_pid
+  # Note: We fork here!
+  repeat "keepalive" &
+  child_pid=$!
+  ensureChildOnEXIT "${child_pid}"
+}
+
+repeat() {
+  local this="$1"
+  while true; do
+    echo "${this}"
+    sleep "${KEEPALIVE_INTERVAL}"
+  done
+}
+
+# Ensures that the child gets killed on normal program exit.
+ensureChildOnEXIT() {
+  # Note: SIGINT and SIGTERM are forwarded to the child process by Bash
+  # automatically, so we don't have to deal with signals.
+
+  local child_pid="$1"
+  trap "kill ${child_pid}" EXIT
+}
+
+main "$@"

cmd/clef/README.md

@@ -661,7 +661,7 @@ OBS! A slight deviation from `json` standard is in place: every request and resp
 Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
 things simpler for both parties.
 
-### ApproveTx
+### ApproveTx / `ui_approveTx`
 
 Invoked when there's a transaction for approval.
 
@@ -673,13 +673,13 @@ Here's a method invocation:
 
 curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
 ```
-
+Results in the following invocation on the UI:
 ```json
 
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -724,7 +724,7 @@ curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","me
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -767,7 +767,7 @@ One which has missing `to`, but with no `data`:
 {
   "jsonrpc": "2.0",
   "id": 3,
-  "method": "ApproveTx",
+  "method": "ui_approveTx",
   "params": [
     {
       "transaction": {
@@ -796,33 +796,7 @@ One which has missing `to`, but with no `data`:
 }
 ```
 
-### ApproveExport
+### ApproveListing / `ui_approveListing`
-
-Invoked when a request to export an account has been made.
-
-#### Sample call
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 7,
-  "method": "ApproveExport",
-  "params": [
-    {
-      "address": "0x0000000000000000000000000000000000000000",
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-
-```
-
-### ApproveListing
 
 Invoked when a request for account listing has been made.
 
@@ -833,7 +807,7 @@ Invoked when a request for account listing has been made.
 {
   "jsonrpc": "2.0",
   "id": 5,
-  "method": "ApproveListing",
+  "method": "ui_approveListing",
   "params": [
     {
       "accounts": [
@@ -860,7 +834,7 @@ Invoked when a request for account listing has been made.
 ```
 
 
-### ApproveSignData
+### ApproveSignData / `ui_approveSignData`
 
 #### Sample call
 
@@ -868,7 +842,7 @@ Invoked when a request for account listing has been made.
 {
   "jsonrpc": "2.0",
   "id": 4,
-  "method": "ApproveSignData",
+  "method": "ui_approveSignData",
   "params": [
     {
       "address": "0x123409812340981234098123409812deadbeef42",
@@ -886,7 +860,7 @@ Invoked when a request for account listing has been made.
 
 ```
 
-### ShowInfo
+### ShowInfo / `ui_showInfo`
 
 The UI should show the info to the user. Does not expect response.
 
@@ -896,7 +870,7 @@ The UI should show the info to the user. Does not expect response.
 {
   "jsonrpc": "2.0",
   "id": 9,
-  "method": "ShowInfo",
+  "method": "ui_showInfo",
   "params": [
     {
       "text": "Tests completed"
@@ -906,7 +880,7 @@ The UI should show the info to the user. Does not expect response.
 
 ```
 
-### ShowError
+### ShowError / `ui_showError`
 
 The UI should show the info to the user. Does not expect response.
 
@@ -925,7 +899,7 @@ The UI should show the info to the user. Does not expect response.
 
 ```
 
-### OnApproved
+### OnApprovedTx / `ui_onApprovedTx`
 
 `OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
 
@@ -933,7 +907,7 @@ When implementing rate-limited rules, this callback should be used.
 
 TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
 
-### OnSignerStartup
+### OnSignerStartup / `ui_onSignerStartup`
 
 This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
 in k/v-form.
@@ -944,7 +918,7 @@ Example call:
 {
   "jsonrpc": "2.0",
   "id": 1,
-  "method": "OnSignerStartup",
+  "method": "ui_onSignerStartup",
   "params": [
     {
       "info": {

cmd/clef/datatypes.md

@@ -35,8 +35,7 @@ Response to SignDataRequest
 Example:
 ```json
 {
-  "approved": true,
+  "approved": true
-  "Password": "apassword"
 }
 ```
 ### SignDataResponse - deny
@@ -46,8 +45,7 @@ Response to SignDataRequest
 Example:
 ```json
 {
-  "approved": false,
+  "approved": false
-  "Password": ""
 }
 ```
 ### SignTxRequest
@@ -89,9 +87,9 @@ Example:
   }
 }
 ```
-### SignDataResponse - approve
+### SignTxResponse - approve
 
-Response to SignDataRequest. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
+Response to request to sign a transaction. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
 
 Example:
 ```json
@@ -105,19 +103,26 @@ Example:
     "nonce": "0x4",
     "data": "0x04030201"
   },
-  "approved": true,
+  "approved": true
-  "password": "apassword"
 }
 ```
-### SignDataResponse - deny
+### SignTxResponse - deny
 
-Response to SignDataRequest. When denying a request, there's no need to provide the transaction in return
+Response to SignTxRequest. When denying a request, there's no need to provide the transaction in return
 
 Example:
 ```json
 {
-  "approved": false,
+  "transaction": {
-  "Password": ""
+    "from": "0x",
+    "to": null,
+    "gas": "0x0",
+    "gasPrice": "0x0",
+    "value": "0x0",
+    "nonce": "0x0",
+    "data": null
+  },
+  "approved": false
 }
 ```
 ### OnApproved - SignTransactionResult
@@ -164,7 +169,7 @@ Example:
 ```
 ### UserInputResponse
 
-Response to SignDataRequest
+Response to UserInputRequest
 
 Example:
 ```json
@@ -198,7 +203,7 @@ Example:
   }
 }
 ```
-### UserInputResponse
+### ListResponse
 
 Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,40 @@
 ### Changelog for internal API (ui-api)
 
+### 6.0.0 
+
+Removed `password` from responses to operations which require them. This is for two reasons, 
+
+- Consistency between how rulesets operate and how manual processing works. A rule can `Approve` but require the actual password to be stored in the clef storage. 
+With this change, the same stored password can be used even if rulesets are not enabled, but storage is. 
+- It also removes the usability-shortcut that a UI might otherwise want to implement; remembering passwords. Since we now will not require the 
+password on every `Approve`, there's no need for the UI to cache it locally. 
+  - In a future update, we'll likely add `clef_storePassword` to the internal API, so the user can store it via his UI (currently only CLI works). 
+
+Affected datatypes:
+- `SignTxResponse`
+- `SignDataResponse`
+- `NewAccountResponse`
+
+If `clef` requires a password, the `OnInputRequired` will be used to collect it. 
+
+
+### 5.0.0 
+
+Changed the namespace format to adhere to the legacy ethereum format: `name_methodName`. Changes:
+
+* `ApproveTx` -> `ui_approveTx`
+* `ApproveSignData` -> `ui_approveSignData`
+* `ApproveExport` -> `removed`
+* `ApproveImport`  -> `removed`
+* `ApproveListing`  -> `ui_approveListing`
+* `ApproveNewAccount`  -> `ui_approveNewAccount`
+* `ShowError` -> `ui_showError`
+* `ShowInfo` -> `ui_showInfo`
+* `OnApprovedTx` -> `ui_onApprovedTx`
+* `OnSignerStartup` -> `ui_onSignerStartup`
+* `OnInputRequired` -> `ui_onInputRequired`
+
+
 ### 4.0.0
 
 * Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:

cmd/clef/main.go

@@ -119,7 +119,7 @@ var (
 	ruleFlag = cli.StringFlag{
 		Name:  "rules",
 		Usage: "Enable rule-engine",
-		Value: "rules.json",
+		Value: "",
 	}
 	stdiouiFlag = cli.BoolFlag{
 		Name: "stdio-ui",
@@ -372,16 +372,13 @@ func signer(c *cli.Context) error {
 
 	var (
 		api       core.ExternalAPI
+		pwStorage storage.Storage = &storage.NoStorage{}
 	)
 
 	configDir := c.GlobalString(configdirFlag.Name)
 	if stretchedKey, err := readMasterKey(c, ui); err != nil {
 		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
-
-		if err != nil {
-			utils.Fatalf(err.Error())
-		}
 		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 
 		// Generate domain specific keys
@@ -390,24 +387,24 @@ func signer(c *cli.Context) error {
 		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
 
 		// Initialize the encrypted storages
-		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		pwStorage = storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
 		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
+		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
+			ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFile))
 			if err != nil {
 				log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 			} else {
-			hasher := sha256.New()
+				shasum := sha256.Sum256(ruleJS)
-			hasher.Write(ruleJS)
+				foundShaSum := hex.EncodeToString(shasum[:])
-			shasum := hasher.Sum(nil)
 				storedShasum := configStorage.Get("ruleset_sha256")
-			if storedShasum != hex.EncodeToString(shasum) {
+				if storedShasum != foundShaSum {
-				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+					log.Info("Could not validate ruleset hash, rules not enabled", "got", foundShaSum, "expected", storedShasum)
 				} else {
 					// Initialize rules
-				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
+					ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage)
 					if err != nil {
 						utils.Fatalf(err.Error())
 					}
@@ -417,6 +414,7 @@ func signer(c *cli.Context) error {
 				}
 			}
 		}
+	}
 	var (
 		chainId  = c.GlobalInt64(chainIdFlag.Name)
 		ksLoc    = c.GlobalString(keystoreFlag.Name)
@@ -427,7 +425,7 @@ func signer(c *cli.Context) error {
 	log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
 		"light-kdf", lightKdf, "advanced", advanced)
 	am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
-	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
+	apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced, pwStorage)
 
 	// Establish the bidirectional communication, by creating a new UI backend and registering
 	// it with the UI.
@@ -533,7 +531,12 @@ func DefaultConfigDir() string {
 		if runtime.GOOS == "darwin" {
 			return filepath.Join(home, "Library", "Signer")
 		} else if runtime.GOOS == "windows" {
+			appdata := os.Getenv("APPDATA")
+			if appdata != "" {
+				return filepath.Join(appdata, "Signer")
+			} else {
 				return filepath.Join(home, "AppData", "Roaming", "Signer")
+			}
 		} else {
 			return filepath.Join(home, ".clef")
 		}
@@ -635,19 +638,46 @@ func testExternalUI(api *core.SignerAPI) {
 	ctx := context.WithValue(context.Background(), "remote", "clef binary")
 	ctx = context.WithValue(ctx, "scheme", "in-proc")
 	ctx = context.WithValue(ctx, "local", "main")
-
 	errs := make([]string, 0)
 
-	api.UI.ShowInfo("Testing 'ShowInfo'")
+	a := common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
-	api.UI.ShowError("Testing 'ShowError'")
 
-	checkErr := func(method string, err error) {
+	queryUser := func(q string) string {
-		if err != nil && err != core.ErrRequestDenied {
+		resp, err := api.UI.OnInputRequired(core.UserInputRequest{
-			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+			Title:  "Testing",
+			Prompt: q,
+		})
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
+		return resp.Text
+	}
+	expectResponse := func(testcase, question, expect string) {
+		if got := queryUser(question); got != expect {
+			errs = append(errs, fmt.Sprintf("%s: got %v, expected %v", testcase, got, expect))
+		}
+	}
+	expectApprove := func(testcase string, err error) {
+		if err == nil || err == accounts.ErrUnknownAccount {
+			return
+		}
+		errs = append(errs, fmt.Sprintf("%v: expected no error, got %v", testcase, err.Error()))
+	}
+	expectDeny := func(testcase string, err error) {
+		if err == nil || err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: expected ErrRequestDenied, got %v", testcase, err))
 		}
 	}
-	var err error
 
+	// Test display of info and error
+	{
+		api.UI.ShowInfo("If you see this message, enter 'yes' to next question")
+		expectResponse("showinfo", "Did you see the message? [yes/no]", "yes")
+		api.UI.ShowError("If you see this message, enter 'yes' to the next question")
+		expectResponse("showerror", "Did you see the message? [yes/no]", "yes")
+	}
+	{ // Sign data test - clique header
+		api.UI.ShowInfo("Please approve the next request for signing a clique header")
 		cliqueHeader := types.Header{
 			common.HexToHash("0000H45H"),
 			common.HexToHash("0000H45H"),
@@ -669,34 +699,63 @@ func testExternalUI(api *core.SignerAPI) {
 		if err != nil {
 			utils.Fatalf("Should not error: %v", err)
 		}
-	addr, err := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
-	if err != nil {
+		_, err = api.SignData(ctx, accounts.MimetypeClique, *addr, hexutil.Encode(cliqueRlp))
-		utils.Fatalf("Should not error: %v", err)
+		expectApprove("signdata - clique header", err)
 	}
-	_, err = api.SignData(ctx, "application/clique", *addr, cliqueRlp)
+	{ // Sign data test - plain text
-	checkErr("SignData", err)
+		api.UI.ShowInfo("Please approve the next request for signing text")
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		_, err := api.SignData(ctx, accounts.MimetypeTextPlain, *addr, hexutil.Encode([]byte("hello world")))
+		expectApprove("signdata - text", err)
+	}
+	{ // Sign data test - plain text reject
+		api.UI.ShowInfo("Please deny the next request for signing text")
+		addr, _ := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
+		_, err := api.SignData(ctx, accounts.MimetypeTextPlain, *addr, hexutil.Encode([]byte("hello world")))
+		expectDeny("signdata - text", err)
+	}
+	{ // Sign transaction
 
-	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
+		api.UI.ShowInfo("Please reject next transaction")
-	checkErr("SignTransaction", err)
+		data := hexutil.Bytes([]byte{})
-	_, err = api.SignData(ctx, "text/plain", common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+		to := common.NewMixedcaseAddress(a)
-	checkErr("SignData", err)
+		tx := core.SendTxArgs{
-	//_, err = api.SignTypedData(ctx, common.MixedcaseAddress{}, core.TypedData{})
+			Data:     &data,
-	//checkErr("SignTypedData", err)
+			Nonce:    0x1,
-	_, err = api.List(ctx)
+			Value:    hexutil.Big(*big.NewInt(6)),
-	checkErr("List", err)
+			From:     common.NewMixedcaseAddress(a),
-	_, err = api.New(ctx)
+			To:       &to,
-	checkErr("New", err)
+			GasPrice: hexutil.Big(*big.NewInt(5)),
+			Gas:      1000,
+			Input:    nil,
+		}
+		_, err := api.SignTransaction(ctx, tx, nil)
+		expectDeny("signtransaction [1]", err)
+		expectResponse("signtransaction [2]", "Did you see any warnings for the last transaction? (yes/no)", "no")
+	}
+	{ // Listing
+		api.UI.ShowInfo("Please reject listing-request")
+		_, err := api.List(ctx)
+		expectDeny("list", err)
+	}
+	{ // Import
+		api.UI.ShowInfo("Please reject new account-request")
+		_, err := api.New(ctx)
+		expectDeny("newaccount", err)
+	}
+	{ // Metadata
+		api.UI.ShowInfo("Please check if you see the Origin in next listing (approve or deny)")
+		api.List(context.WithValue(ctx, "Origin", "origin.com"))
+		expectResponse("metadata - origin", "Did you see origin (origin.com)? [yes/no] ", "yes")
+	}
 
-	api.UI.ShowInfo("Tests completed")
-
-	if len(errs) > 0 {
-		log.Error("Got errors")
 	for _, e := range errs {
 		log.Error(e)
 	}
-	} else {
+	result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n"))
-		log.Info("No errors")
+	api.UI.ShowInfo(result)
-	}
+
 }
 
 // getPassPhrase retrieves the password associated with clef, either fetched
@@ -793,7 +852,7 @@ func GenDoc(ctx *cli.Context) {
 	}
 	{ // Sign plain text response
 		add("SignDataResponse - approve", "Response to SignDataRequest",
-			&core.SignDataResponse{Password: "apassword", Approved: true})
+			&core.SignDataResponse{Approved: true})
 		add("SignDataResponse - deny", "Response to SignDataRequest",
 			&core.SignDataResponse{})
 	}
@@ -828,9 +887,9 @@ func GenDoc(ctx *cli.Context) {
 	}
 	{ // Sign tx response
 		data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
-		add("SignDataResponse - approve", "Response to SignDataRequest. This response needs to contain the `transaction`"+
+		add("SignTxResponse - approve", "Response to request to sign a transaction. This response needs to contain the `transaction`"+
 			", because the UI is free to make modifications to the transaction.",
-			&core.SignTxResponse{Password: "apassword", Approved: true,
+			&core.SignTxResponse{Approved: true,
 				Transaction: core.SendTxArgs{
 					Data:     &data,
 					Nonce:    0x4,
@@ -841,9 +900,9 @@ func GenDoc(ctx *cli.Context) {
 					Gas:      1000,
 					Input:    nil,
 				}})
-		add("SignDataResponse - deny", "Response to SignDataRequest. When denying a request, there's no need to "+
+		add("SignTxResponse - deny", "Response to SignTxRequest. When denying a request, there's no need to "+
 			"provide the transaction in return",
-			&core.SignDataResponse{})
+			&core.SignTxResponse{})
 	}
 	{ // WHen a signed tx is ready to go out
 		desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
@@ -869,7 +928,7 @@ func GenDoc(ctx *cli.Context) {
 	{ // User input
 		add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
 			&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
-		add("UserInputResponse", "Response to SignDataRequest",
+		add("UserInputResponse", "Response to UserInputRequest",
 			&core.UserInputResponse{Text: "The textual response from user"})
 	}
 	{ // List request
@@ -883,7 +942,7 @@ func GenDoc(ctx *cli.Context) {
 					{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
 			})
 
-		add("UserInputResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
+		add("ListResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
 			"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
 			&core.ListResponse{
 				Accounts: []accounts.Account{

cmd/evm/internal/compiler/compiler.go

@@ -25,7 +25,7 @@ import (
 
 func Compile(fn string, src []byte, debug bool) (string, error) {
 	compiler := asm.NewCompiler(debug)
-	compiler.Feed(asm.Lex(fn, src, debug))
+	compiler.Feed(asm.Lex(src, debug))
 
 	bin, compileErrors := compiler.Compile()
 	if len(compileErrors) > 0 {

cmd/evm/runner.go

@@ -31,10 +31,10 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	cli "gopkg.in/urfave/cli.v1"
@@ -99,12 +99,12 @@ func runCmd(ctx *cli.Context) error {
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
 		genesisConfig = gen
-		db := ethdb.NewMemDatabase()
+		db := rawdb.NewMemoryDatabase()
 		genesis := gen.ToBlock(db)
 		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
 	} else {
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 		genesisConfig = new(core.Genesis)
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {

cmd/geth/chaincmd.go

@@ -29,14 +29,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -193,7 +191,7 @@ func initGenesis(ctx *cli.Context) error {
 	defer stack.Close()
 
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		chaindb, err := stack.OpenDatabase(name, 0, 0)
+		chaindb, err := stack.OpenDatabase(name, 0, 0, "")
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}
@@ -201,6 +199,7 @@ func initGenesis(ctx *cli.Context) error {
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
+		chaindb.Close()
 		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
 	return nil
@@ -213,8 +212,8 @@ func importChain(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
 	defer stack.Close()
 
-	chain, chainDb := utils.MakeChain(ctx, stack)
+	chain, db := utils.MakeChain(ctx, stack)
-	defer chainDb.Close()
+	defer db.Close()
 
 	// Start periodically gathering memory profiles
 	var peakMemAlloc, peakMemSys uint64
@@ -249,23 +248,18 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing
-	db := chainDb.(*ethdb.LDBDatabase)
+	stats, err := db.Stat("leveldb.stats")
-
-	stats, err := db.LDB().GetProperty("leveldb.stats")
 	if err != nil {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
 
-	ioStats, err := db.LDB().GetProperty("leveldb.iostats")
+	ioStats, err := db.Stat("leveldb.iostats")
 	if err != nil {
 		utils.Fatalf("Failed to read database iostats: %v", err)
 	}
 	fmt.Println(ioStats)
 
-	fmt.Printf("Trie cache misses:  %d\n", trie.CacheMisses())
-	fmt.Printf("Trie cache unloads: %d\n\n", trie.CacheUnloads())
-
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)
 	runtime.ReadMemStats(mem)
@@ -282,23 +276,22 @@ func importChain(ctx *cli.Context) error {
 	// Compact the entire database to more accurately measure disk io and print the stats
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
-	if err = db.LDB().CompactRange(util.Range{}); err != nil {
+	if err = db.Compact(nil, nil); err != nil {
 		utils.Fatalf("Compaction failed: %v", err)
 	}
 	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
 
-	stats, err = db.LDB().GetProperty("leveldb.stats")
+	stats, err = db.Stat("leveldb.stats")
 	if err != nil {
 		utils.Fatalf("Failed to read database stats: %v", err)
 	}
 	fmt.Println(stats)
 
-	ioStats, err = db.LDB().GetProperty("leveldb.iostats")
+	ioStats, err = db.Stat("leveldb.iostats")
 	if err != nil {
 		utils.Fatalf("Failed to read database iostats: %v", err)
 	}
 	fmt.Println(ioStats)
-
 	return nil
 }
 
@@ -344,10 +337,10 @@ func importPreimages(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
 	defer stack.Close()
 
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	db := utils.MakeChainDatabase(ctx, stack)
 	start := time.Now()
 
-	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
+	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
 		utils.Fatalf("Import error: %v\n", err)
 	}
 	fmt.Printf("Import done in %v\n", time.Since(start))
@@ -362,10 +355,10 @@ func exportPreimages(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
 	defer stack.Close()
 
-	diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
+	db := utils.MakeChainDatabase(ctx, stack)
 	start := time.Now()
 
-	if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
+	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
 	fmt.Printf("Export done in %v\n", time.Since(start))
@@ -386,7 +379,7 @@ func copyDb(ctx *cli.Context) error {
 	dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
 
 	// Create a source peer to satisfy downloader requests from
-	db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256)
+	db, err := rawdb.NewLevelDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256, "")
 	if err != nil {
 		return err
 	}
@@ -413,11 +406,10 @@ func copyDb(ctx *cli.Context) error {
 	// Compact the entire database to remove any sync overhead
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
-	if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil {
+	if err = db.Compact(nil, nil); err != nil {
 		utils.Fatalf("Compaction failed: %v", err)
 	}
 	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
-
 	return nil
 }
 

cmd/geth/dao_test.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -121,7 +120,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "geth", "chaindata")
-	db, err := ethdb.NewLDBDatabase(path, 0, 0)
+	db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
 	}

cmd/geth/main.go

@@ -93,6 +93,8 @@ var (
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.LightServFlag,
+		utils.LightBandwidthInFlag,
+		utils.LightBandwidthOutFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
 		utils.WhitelistFlag,
@@ -100,7 +102,6 @@ var (
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
-		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,

cmd/geth/usage.go

@@ -81,6 +81,8 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
 			utils.LightServFlag,
+			utils.LightBandwidthInFlag,
+			utils.LightBandwidthOutFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,
@@ -137,7 +139,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.CacheDatabaseFlag,
 			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
-			utils.TrieCacheGenFlag,
 		},
 	},
 	{

cmd/swarm/config.go

@@ -59,31 +59,31 @@ var (
 
 //constants for environment variables
 const (
-	SWARM_ENV_CHEQUEBOOK_ADDR         = "SWARM_CHEQUEBOOK_ADDR"
+	SwarmEnvChequebookAddr       = "SWARM_CHEQUEBOOK_ADDR"
-	SWARM_ENV_ACCOUNT                 = "SWARM_ACCOUNT"
+	SwarmEnvAccount              = "SWARM_ACCOUNT"
-	SWARM_ENV_LISTEN_ADDR             = "SWARM_LISTEN_ADDR"
+	SwarmEnvListenAddr           = "SWARM_LISTEN_ADDR"
-	SWARM_ENV_PORT                    = "SWARM_PORT"
+	SwarmEnvPort                 = "SWARM_PORT"
-	SWARM_ENV_NETWORK_ID              = "SWARM_NETWORK_ID"
+	SwarmEnvNetworkID            = "SWARM_NETWORK_ID"
-	SWARM_ENV_SWAP_ENABLE             = "SWARM_SWAP_ENABLE"
+	SwarmEnvSwapEnable           = "SWARM_SWAP_ENABLE"
-	SWARM_ENV_SWAP_API                = "SWARM_SWAP_API"
+	SwarmEnvSwapAPI              = "SWARM_SWAP_API"
-	SWARM_ENV_SYNC_DISABLE            = "SWARM_SYNC_DISABLE"
+	SwarmEnvSyncDisable          = "SWARM_SYNC_DISABLE"
-	SWARM_ENV_SYNC_UPDATE_DELAY       = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SwarmEnvSyncUpdateDelay      = "SWARM_ENV_SYNC_UPDATE_DELAY"
-	SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
+	SwarmEnvMaxStreamPeerServers = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
-	SWARM_ENV_LIGHT_NODE_ENABLE       = "SWARM_LIGHT_NODE_ENABLE"
+	SwarmEnvLightNodeEnable      = "SWARM_LIGHT_NODE_ENABLE"
-	SWARM_ENV_DELIVERY_SKIP_CHECK     = "SWARM_DELIVERY_SKIP_CHECK"
+	SwarmEnvDeliverySkipCheck    = "SWARM_DELIVERY_SKIP_CHECK"
-	SWARM_ENV_ENS_API                 = "SWARM_ENS_API"
+	SwarmEnvENSAPI               = "SWARM_ENS_API"
-	SWARM_ENV_ENS_ADDR                = "SWARM_ENS_ADDR"
+	SwarmEnvENSAddr              = "SWARM_ENS_ADDR"
-	SWARM_ENV_CORS                    = "SWARM_CORS"
+	SwarmEnvCORS                 = "SWARM_CORS"
-	SWARM_ENV_BOOTNODES               = "SWARM_BOOTNODES"
+	SwarmEnvBootnodes            = "SWARM_BOOTNODES"
-	SWARM_ENV_PSS_ENABLE              = "SWARM_PSS_ENABLE"
+	SwarmEnvPSSEnable            = "SWARM_PSS_ENABLE"
-	SWARM_ENV_STORE_PATH              = "SWARM_STORE_PATH"
+	SwarmEnvStorePath            = "SWARM_STORE_PATH"
-	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
+	SwarmEnvStoreCapacity        = "SWARM_STORE_CAPACITY"
-	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
+	SwarmEnvStoreCacheCapacity   = "SWARM_STORE_CACHE_CAPACITY"
-	SWARM_ENV_BOOTNODE_MODE           = "SWARM_BOOTNODE_MODE"
+	SwarmEnvBootnodeMode         = "SWARM_BOOTNODE_MODE"
-	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	SwarmAccessPassword          = "SWARM_ACCESS_PASSWORD"
-	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
+	SwarmAutoDefaultPath         = "SWARM_AUTO_DEFAULTPATH"
-	SWARM_GLOBALSTORE_API             = "SWARM_GLOBALSTORE_API"
+	SwarmGlobalstoreAPI          = "SWARM_GLOBALSTORE_API"
-	GETH_ENV_DATADIR                  = "GETH_DATADIR"
+	GethEnvDataDir               = "GETH_DATADIR"
 )
 
 // These settings ensure that TOML keys use the same names as Go struct fields.
@@ -227,7 +227,7 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 
 	currentConfig.SwapAPI = ctx.GlobalString(SwarmSwapAPIFlag.Name)
 	if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
-		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
+		utils.Fatalf(SwarmErrSwapSetNoAPI)
 	}
 
 	if ctx.GlobalIsSet(EnsAPIFlag.Name) {
@@ -255,8 +255,8 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.LocalStoreParams.DbCapacity = storeCapacity
 	}
 
-	if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
+	if ctx.GlobalIsSet(SwarmStoreCacheCapacity.Name) {
-		currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity
+		currentConfig.LocalStoreParams.CacheCapacity = ctx.GlobalUint(SwarmStoreCacheCapacity.Name)
 	}
 
 	if ctx.GlobalIsSet(SwarmBootnodeModeFlag.Name) {
@@ -274,113 +274,113 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 // envVarsOverride overrides the current config with whatver is provided in environment variables
 // most values are not allowed a zero value (empty string), if not otherwise noted
 func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
-	if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" {
+	if keyid := os.Getenv(SwarmEnvAccount); keyid != "" {
 		currentConfig.BzzAccount = keyid
 	}
 
-	if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" {
+	if chbookaddr := os.Getenv(SwarmEnvChequebookAddr); chbookaddr != "" {
 		currentConfig.Contract = common.HexToAddress(chbookaddr)
 	}
 
-	if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
+	if networkid := os.Getenv(SwarmEnvNetworkID); networkid != "" {
 		id, err := strconv.ParseUint(networkid, 10, 64)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvNetworkID, err)
 		}
 		if id != 0 {
 			currentConfig.NetworkID = id
 		}
 	}
 
-	if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
+	if datadir := os.Getenv(GethEnvDataDir); datadir != "" {
 		currentConfig.Path = expandPath(datadir)
 	}
 
-	bzzport := os.Getenv(SWARM_ENV_PORT)
+	bzzport := os.Getenv(SwarmEnvPort)
 	if len(bzzport) > 0 {
 		currentConfig.Port = bzzport
 	}
 
-	if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" {
+	if bzzaddr := os.Getenv(SwarmEnvListenAddr); bzzaddr != "" {
 		currentConfig.ListenAddr = bzzaddr
 	}
 
-	if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
+	if swapenable := os.Getenv(SwarmEnvSwapEnable); swapenable != "" {
 		swap, err := strconv.ParseBool(swapenable)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSwapEnable, err)
 		}
 		currentConfig.SwapEnabled = swap
 	}
 
-	if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
+	if syncdisable := os.Getenv(SwarmEnvSyncDisable); syncdisable != "" {
 		sync, err := strconv.ParseBool(syncdisable)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncDisable, err)
 		}
 		currentConfig.SyncEnabled = !sync
 	}
 
-	if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
+	if v := os.Getenv(SwarmEnvDeliverySkipCheck); v != "" {
 		skipCheck, err := strconv.ParseBool(v)
 		if err != nil {
 			currentConfig.DeliverySkipCheck = skipCheck
 		}
 	}
 
-	if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
+	if v := os.Getenv(SwarmEnvSyncUpdateDelay); v != "" {
 		d, err := time.ParseDuration(v)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncUpdateDelay, err)
 		}
 		currentConfig.SyncUpdateDelay = d
 	}
 
-	if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
+	if max := os.Getenv(SwarmEnvMaxStreamPeerServers); max != "" {
 		m, err := strconv.Atoi(max)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvMaxStreamPeerServers, err)
 		}
 		currentConfig.MaxStreamPeerServers = m
 	}
 
-	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
+	if lne := os.Getenv(SwarmEnvLightNodeEnable); lne != "" {
 		lightnode, err := strconv.ParseBool(lne)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvLightNodeEnable, err)
 		}
 		currentConfig.LightNodeEnabled = lightnode
 	}
 
-	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
+	if swapapi := os.Getenv(SwarmEnvSwapAPI); swapapi != "" {
 		currentConfig.SwapAPI = swapapi
 	}
 
 	if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
-		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
+		utils.Fatalf(SwarmErrSwapSetNoAPI)
 	}
 
-	if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" {
+	if ensapi := os.Getenv(SwarmEnvENSAPI); ensapi != "" {
 		currentConfig.EnsAPIs = strings.Split(ensapi, ",")
 	}
 
-	if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
+	if ensaddr := os.Getenv(SwarmEnvENSAddr); ensaddr != "" {
 		currentConfig.EnsRoot = common.HexToAddress(ensaddr)
 	}
 
-	if cors := os.Getenv(SWARM_ENV_CORS); cors != "" {
+	if cors := os.Getenv(SwarmEnvCORS); cors != "" {
 		currentConfig.Cors = cors
 	}
 
-	if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" {
+	if bm := os.Getenv(SwarmEnvBootnodeMode); bm != "" {
 		bootnodeMode, err := strconv.ParseBool(bm)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmEnvBootnodeMode, err)
 		}
 		currentConfig.BootnodeMode = bootnodeMode
 	}
 
-	if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" {
+	if api := os.Getenv(SwarmGlobalstoreAPI); api != "" {
 		currentConfig.GlobalStoreAPI = api
 	}
 

cmd/swarm/config_test.go

@@ -52,7 +52,7 @@ func TestConfigFailsSwapEnabledNoSwapApi(t *testing.T) {
 	}
 
 	swarm := runSwarm(t, flags...)
-	swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n")
+	swarm.Expect("Fatal: " + SwarmErrSwapSetNoAPI + "\n")
 	swarm.ExpectExit()
 }
 
@@ -63,7 +63,7 @@ func TestConfigFailsNoBzzAccount(t *testing.T) {
 	}
 
 	swarm := runSwarm(t, flags...)
-	swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n")
+	swarm.Expect("Fatal: " + SwarmErrNoBZZAccount + "\n")
 	swarm.ExpectExit()
 }
 

cmd/swarm/flags.go

@@ -23,68 +23,68 @@ var (
 	ChequebookAddrFlag = cli.StringFlag{
 		Name:   "chequebook",
 		Usage:  "chequebook contract address",
-		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
+		EnvVar: SwarmEnvChequebookAddr,
 	}
 	SwarmAccountFlag = cli.StringFlag{
 		Name:   "bzzaccount",
 		Usage:  "Swarm account key file",
-		EnvVar: SWARM_ENV_ACCOUNT,
+		EnvVar: SwarmEnvAccount,
 	}
 	SwarmListenAddrFlag = cli.StringFlag{
 		Name:   "httpaddr",
 		Usage:  "Swarm HTTP API listening interface",
-		EnvVar: SWARM_ENV_LISTEN_ADDR,
+		EnvVar: SwarmEnvListenAddr,
 	}
 	SwarmPortFlag = cli.StringFlag{
 		Name:   "bzzport",
 		Usage:  "Swarm local http api port",
-		EnvVar: SWARM_ENV_PORT,
+		EnvVar: SwarmEnvPort,
 	}
 	SwarmNetworkIdFlag = cli.IntFlag{
 		Name:   "bzznetworkid",
 		Usage:  "Network identifier (integer, default 3=swarm testnet)",
-		EnvVar: SWARM_ENV_NETWORK_ID,
+		EnvVar: SwarmEnvNetworkID,
 	}
 	SwarmSwapEnabledFlag = cli.BoolFlag{
 		Name:   "swap",
 		Usage:  "Swarm SWAP enabled (default false)",
-		EnvVar: SWARM_ENV_SWAP_ENABLE,
+		EnvVar: SwarmEnvSwapEnable,
 	}
 	SwarmSwapAPIFlag = cli.StringFlag{
 		Name:   "swap-api",
 		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
-		EnvVar: SWARM_ENV_SWAP_API,
+		EnvVar: SwarmEnvSwapAPI,
 	}
 	SwarmSyncDisabledFlag = cli.BoolTFlag{
 		Name:   "nosync",
 		Usage:  "Disable swarm syncing",
-		EnvVar: SWARM_ENV_SYNC_DISABLE,
+		EnvVar: SwarmEnvSyncDisable,
 	}
 	SwarmSyncUpdateDelay = cli.DurationFlag{
 		Name:   "sync-update-delay",
 		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
-		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
+		EnvVar: SwarmEnvSyncUpdateDelay,
 	}
 	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
 		Name:   "max-stream-peer-servers",
 		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
-		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		EnvVar: SwarmEnvMaxStreamPeerServers,
 		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
 	}
 	SwarmLightNodeEnabled = cli.BoolFlag{
 		Name:   "lightnode",
 		Usage:  "Enable Swarm LightNode (default false)",
-		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+		EnvVar: SwarmEnvLightNodeEnable,
 	}
 	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
 		Name:   "delivery-skip-check",
 		Usage:  "Skip chunk delivery check (default false)",
-		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
+		EnvVar: SwarmEnvDeliverySkipCheck,
 	}
 	EnsAPIFlag = cli.StringSliceFlag{
 		Name:   "ens-api",
 		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
-		EnvVar: SWARM_ENV_ENS_API,
+		EnvVar: SwarmEnvENSAPI,
 	}
 	SwarmApiFlag = cli.StringFlag{
 		Name:  "bzzapi",
@@ -126,7 +126,7 @@ var (
 	SwarmAccessPasswordFlag = cli.StringFlag{
 		Name:   "password",
 		Usage:  "Password",
-		EnvVar: SWARM_ACCESS_PASSWORD,
+		EnvVar: SwarmAccessPassword,
 	}
 	SwarmDryRunFlag = cli.BoolFlag{
 		Name:  "dry-run",
@@ -135,22 +135,23 @@ var (
 	CorsStringFlag = cli.StringFlag{
 		Name:   "corsdomain",
 		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
-		EnvVar: SWARM_ENV_CORS,
+		EnvVar: SwarmEnvCORS,
 	}
 	SwarmStorePath = cli.StringFlag{
 		Name:   "store.path",
 		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
-		EnvVar: SWARM_ENV_STORE_PATH,
+		EnvVar: SwarmEnvStorePath,
 	}
 	SwarmStoreCapacity = cli.Uint64Flag{
 		Name:   "store.size",
 		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
-		EnvVar: SWARM_ENV_STORE_CAPACITY,
+		EnvVar: SwarmEnvStoreCapacity,
 	}
 	SwarmStoreCacheCapacity = cli.UintFlag{
 		Name:   "store.cache.size",
-		Usage:  "Number of recent chunks cached in memory (default 5000)",
+		Usage:  "Number of recent chunks cached in memory",
-		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
+		EnvVar: SwarmEnvStoreCacheCapacity,
+		Value:  10000,
 	}
 	SwarmCompressedFlag = cli.BoolFlag{
 		Name:  "compressed",
@@ -179,6 +180,6 @@ var (
 	SwarmGlobalStoreAPIFlag = cli.StringFlag{
 		Name:   "globalstore-api",
 		Usage:  "URL of the Global Store API provider (only for testing)",
-		EnvVar: SWARM_GLOBALSTORE_API,
+		EnvVar: SwarmGlobalstoreAPI,
 	}
 )

cmd/swarm/global-store/explorer.go

@@ -0,0 +1,66 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// serveChunkExplorer starts an http server in background with chunk explorer handler
+// using the provided global store. Server is started if the returned shutdown function
+// is not nil.
+func serveChunkExplorer(ctx *cli.Context, globalStore mock.GlobalStorer) (shutdown func(), err error) {
+	if !ctx.IsSet("explorer-address") {
+		return nil, nil
+	}
+
+	corsOrigins := ctx.StringSlice("explorer-cors-origin")
+	server := &http.Server{
+		Handler:      explorer.NewHandler(globalStore, corsOrigins),
+		IdleTimeout:  30 * time.Minute,
+		ReadTimeout:  2 * time.Minute,
+		WriteTimeout: 2 * time.Minute,
+	}
+	listener, err := net.Listen("tcp", ctx.String("explorer-address"))
+	if err != nil {
+		return nil, fmt.Errorf("explorer: %v", err)
+	}
+	log.Info("chunk explorer http", "address", listener.Addr().String(), "origins", corsOrigins)
+
+	go func() {
+		if err := server.Serve(listener); err != nil {
+			log.Error("chunk explorer", "err", err)
+		}
+	}()
+
+	return func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := server.Shutdown(ctx); err != nil {
+			log.Error("chunk explorer: shutdown", "err", err)
+		}
+	}, nil
+}

cmd/swarm/global-store/explorer_test.go

@@ -0,0 +1,254 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
+	mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
+)
+
+// TestExplorer validates basic chunk explorer functionality by storing
+// a small set of chunk and making http requests on exposed endpoint.
+// Full chunk explorer validation is done in mock/explorer package.
+func TestExplorer(t *testing.T) {
+	addr := findFreeTCPAddress(t)
+	explorerAddr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, "ws", "--addr", addr, "--explorer-address", explorerAddr)
+	defer testCmd.Kill()
+
+	client := websocketClient(t, addr)
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	nodeKeys := map[string][]string{
+		"a1": {"b1", "b2", "b3"},
+		"a2": {"b3", "b4", "b5"},
+	}
+
+	keyNodes := make(map[string][]string)
+
+	for addr, keys := range nodeKeys {
+		for _, key := range keys {
+			keyNodes[key] = append(keyNodes[key], addr)
+		}
+	}
+
+	invalidAddr := "c1"
+	invalidKey := "d1"
+
+	for addr, keys := range nodeKeys {
+		for _, key := range keys {
+			err := store.Put(common.HexToAddress(addr), common.Hex2Bytes(key), []byte("data"))
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	endpoint := "http://" + explorerAddr
+
+	t.Run("has key", func(t *testing.T) {
+		for addr, keys := range nodeKeys {
+			for _, key := range keys {
+				testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+key, http.StatusOK)
+				testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+key, http.StatusNotFound)
+			}
+			testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+invalidKey, http.StatusNotFound)
+		}
+		testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+invalidKey, http.StatusNotFound)
+	})
+
+	t.Run("keys", func(t *testing.T) {
+		var keys []string
+		for key := range keyNodes {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		testKeysResponse(t, endpoint+"/api/keys", explorer.KeysResponse{
+			Keys: keys,
+		})
+	})
+
+	t.Run("nodes", func(t *testing.T) {
+		var nodes []string
+		for addr := range nodeKeys {
+			nodes = append(nodes, common.HexToAddress(addr).Hex())
+		}
+		sort.Strings(nodes)
+		testNodesResponse(t, endpoint+"/api/nodes", explorer.NodesResponse{
+			Nodes: nodes,
+		})
+	})
+
+	t.Run("node keys", func(t *testing.T) {
+		for addr, keys := range nodeKeys {
+			testKeysResponse(t, endpoint+"/api/keys?node="+addr, explorer.KeysResponse{
+				Keys: keys,
+			})
+		}
+		testKeysResponse(t, endpoint+"/api/keys?node="+invalidAddr, explorer.KeysResponse{})
+	})
+
+	t.Run("key nodes", func(t *testing.T) {
+		for key, addrs := range keyNodes {
+			var nodes []string
+			for _, addr := range addrs {
+				nodes = append(nodes, common.HexToAddress(addr).Hex())
+			}
+			sort.Strings(nodes)
+			testNodesResponse(t, endpoint+"/api/nodes?key="+key, explorer.NodesResponse{
+				Nodes: nodes,
+			})
+		}
+		testNodesResponse(t, endpoint+"/api/nodes?key="+invalidKey, explorer.NodesResponse{})
+	})
+}
+
+// TestExplorer_CORSOrigin validates if chunk explorer returns
+// correct CORS origin header in GET and OPTIONS requests.
+func TestExplorer_CORSOrigin(t *testing.T) {
+	origin := "http://localhost/"
+	addr := findFreeTCPAddress(t)
+	explorerAddr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, "ws",
+		"--addr", addr,
+		"--explorer-address", explorerAddr,
+		"--explorer-cors-origin", origin,
+	)
+	defer testCmd.Kill()
+
+	// wait until the server is started
+	waitHTTPEndpoint(t, explorerAddr)
+
+	url := "http://" + explorerAddr + "/api/keys"
+
+	t.Run("get", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Set("Origin", origin)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		header := resp.Header.Get("Access-Control-Allow-Origin")
+		if header != origin {
+			t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
+		}
+	})
+
+	t.Run("preflight", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodOptions, url, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Set("Origin", origin)
+		req.Header.Set("Access-Control-Request-Method", "GET")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		header := resp.Header.Get("Access-Control-Allow-Origin")
+		if header != origin {
+			t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
+		}
+	})
+}
+
+// testStatusResponse makes an http request to provided url
+// and validates if response is explorer.StatusResponse for
+// the expected status code.
+func testStatusResponse(t *testing.T, url string, code int) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != code {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, code)
+	}
+	var r explorer.StatusResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if r.Code != code {
+		t.Errorf("got response code %v, want %v", r.Code, code)
+	}
+	if r.Message != http.StatusText(code) {
+		t.Errorf("got response message %q, want %q", r.Message, http.StatusText(code))
+	}
+}
+
+// testKeysResponse makes an http request to provided url
+// and validates if response machhes expected explorer.KeysResponse.
+func testKeysResponse(t *testing.T, url string, want explorer.KeysResponse) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
+	}
+	var r explorer.KeysResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprint(r.Keys) != fmt.Sprint(want.Keys) {
+		t.Errorf("got keys %v, want %v", r.Keys, want.Keys)
+	}
+	if r.Next != want.Next {
+		t.Errorf("got next %s, want %s", r.Next, want.Next)
+	}
+}
+
+// testNodeResponse makes an http request to provided url
+// and validates if response machhes expected explorer.NodeResponse.
+func testNodesResponse(t *testing.T, url string, want explorer.NodesResponse) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
+	}
+	var r explorer.NodesResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprint(r.Nodes) != fmt.Sprint(want.Nodes) {
+		t.Errorf("got nodes %v, want %v", r.Nodes, want.Nodes)
+	}
+	if r.Next != want.Next {
+		t.Errorf("got next %s, want %s", r.Next, want.Next)
+	}
+}

cmd/swarm/global-store/global_store.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -66,7 +67,7 @@ func startWS(ctx *cli.Context) (err error) {
 	return http.Serve(listener, server.WebsocketHandler(origins))
 }
 
-// newServer creates a global store and returns its RPC server.
+// newServer creates a global store and starts a chunk explorer server if configured.
 // Returned cleanup function should be called only if err is nil.
 func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
 	log.PrintOrigins(true)
@@ -81,7 +82,9 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error)
 			return nil, nil, err
 		}
 		cleanup = func() {
-			dbStore.Close()
+			if err := dbStore.Close(); err != nil {
+				log.Error("global store: close", "err", err)
+			}
 		}
 		globalStore = dbStore
 		log.Info("database global store", "dir", dir)
@@ -96,5 +99,22 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error)
 		return nil, nil, err
 	}
 
+	shutdown, err := serveChunkExplorer(ctx, globalStore)
+	if err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+	if shutdown != nil {
+		cleanup = func() {
+			shutdown()
+
+			if c, ok := globalStore.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Error("global store: close", "err", err)
+				}
+			}
+		}
+	}
+
 	return server, cleanup, nil
 }

cmd/swarm/global-store/global_store_test.go

@@ -60,7 +60,7 @@ func TestHTTP_Database(t *testing.T) {
 func testHTTP(t *testing.T, put bool, args ...string) {
 	addr := findFreeTCPAddress(t)
 	testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
-	defer testCmd.Interrupt()
+	defer testCmd.Kill()
 
 	client, err := rpc.DialHTTP("http://" + addr)
 	if err != nil {
@@ -69,16 +69,7 @@ func testHTTP(t *testing.T, put bool, args ...string) {
 
 	// wait until global store process is started as
 	// rpc.DialHTTP is actually not connecting
-	for i := 0; i < 1000; i++ {
+	waitHTTPEndpoint(t, addr)
-		_, err = http.DefaultClient.Get("http://" + addr)
-		if err == nil {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	store := mockRPC.NewGlobalStore(client)
 	defer store.Close()
@@ -135,21 +126,9 @@ func TestWebsocket_Database(t *testing.T) {
 func testWebsocket(t *testing.T, put bool, args ...string) {
 	addr := findFreeTCPAddress(t)
 	testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
-	defer testCmd.Interrupt()
+	defer testCmd.Kill()
 
-	var client *rpc.Client
+	client := websocketClient(t, addr)
-	var err error
-	// wait until global store process is started
-	for i := 0; i < 1000; i++ {
-		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
-		if err == nil {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	store := mockRPC.NewGlobalStore(client)
 	defer store.Close()
@@ -160,7 +139,7 @@ func testWebsocket(t *testing.T, put bool, args ...string) {
 	wantValue := "value"
 
 	if put {
-		err = node.Put([]byte(wantKey), []byte(wantValue))
+		err := node.Put([]byte(wantKey), []byte(wantValue))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -189,3 +168,40 @@ func findFreeTCPAddress(t *testing.T) (addr string) {
 
 	return listener.Addr().String()
 }
+
+// websocketClient waits until global store process is started
+// and returns rpc client.
+func websocketClient(t *testing.T, addr string) (client *rpc.Client) {
+	t.Helper()
+
+	var err error
+	for i := 0; i < 1000; i++ {
+		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return client
+}
+
+// waitHTTPEndpoint retries http requests to a provided
+// address until the connection is established.
+func waitHTTPEndpoint(t *testing.T, addr string) {
+	t.Helper()
+
+	var err error
+	for i := 0; i < 1000; i++ {
+		_, err = http.Get("http://" + addr)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+}

cmd/swarm/global-store/main.go

@@ -19,12 +19,14 @@ package main
 import (
 	"os"
 
-	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
-var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+var (
+	version   = "0.1"
+	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+)
 
 func main() {
 	err := newApp().Run(os.Args)
@@ -37,16 +39,30 @@ func main() {
 // newApp construct a new instance of Swarm Global Store.
 // Method Run is called on it in the main function and in tests.
 func newApp() (app *cli.App) {
-	app = utils.NewApp(gitCommit, "Swarm Global Store")
+	app = cli.NewApp()
-
 	app.Name = "global-store"
+	app.Version = version
+	if len(gitCommit) >= 8 {
+		app.Version += "-" + gitCommit[:8]
+	}
+	app.Usage = "Swarm Global Store"
 
 	// app flags (for all commands)
 	app.Flags = []cli.Flag{
 		cli.IntFlag{
 			Name:  "verbosity",
 			Value: 3,
-			Usage: "verbosity level",
+			Usage: "Verbosity level.",
+		},
+		cli.StringFlag{
+			Name:  "explorer-address",
+			Value: "",
+			Usage: "Chunk explorer HTTP listener address.",
+		},
+		cli.StringSliceFlag{
+			Name:  "explorer-cors-origin",
+			Value: nil,
+			Usage: "Chunk explorer CORS origin (can be specified multiple times).",
 		},
 	}
 
@@ -54,7 +70,7 @@ func newApp() (app *cli.App) {
 		{
 			Name:    "http",
 			Aliases: []string{"h"},
-			Usage:   "start swarm global store with http server",
+			Usage:   "Start swarm global store with HTTP server.",
 			Action:  startHTTP,
 			// Flags only for "start" command.
 			// Allow app flags to be specified after the
@@ -63,19 +79,19 @@ func newApp() (app *cli.App) {
 				cli.StringFlag{
 					Name:  "dir",
 					Value: "",
-					Usage: "data directory",
+					Usage: "Data directory.",
 				},
 				cli.StringFlag{
 					Name:  "addr",
 					Value: "0.0.0.0:3033",
-					Usage: "address to listen for http connection",
+					Usage: "Address to listen for HTTP connections.",
 				},
 			),
 		},
 		{
 			Name:    "websocket",
 			Aliases: []string{"ws"},
-			Usage:   "start swarm global store with websocket server",
+			Usage:   "Start swarm global store with WebSocket server.",
 			Action:  startWS,
 			// Flags only for "start" command.
 			// Allow app flags to be specified after the
@@ -84,17 +100,17 @@ func newApp() (app *cli.App) {
 				cli.StringFlag{
 					Name:  "dir",
 					Value: "",
-					Usage: "data directory",
+					Usage: "Data directory.",
 				},
 				cli.StringFlag{
 					Name:  "addr",
 					Value: "0.0.0.0:3033",
-					Usage: "address to listen for websocket connection",
+					Usage: "Address to listen for WebSocket connections.",
 				},
 				cli.StringSliceFlag{
-					Name:  "origins",
+					Name:  "origin",
-					Value: &cli.StringSlice{"*"},
+					Value: nil,
-					Usage: "websocket origins",
+					Usage: "WebSocket CORS origin (can be specified multiple times).",
 				},
 			),
 		},

cmd/swarm/main.go

@@ -76,8 +76,8 @@ var gitCommit string
 
 //declare a few constant error messages, useful for later error check comparisons in test
 var (
-	SWARM_ERR_NO_BZZACCOUNT   = "bzzaccount option is required but not set; check your config file, command line or environment variables"
+	SwarmErrNoBZZAccount = "bzzaccount option is required but not set; check your config file, command line or environment variables"
-	SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set"
+	SwarmErrSwapSetNoAPI = "SWAP is enabled but --swap-api is not set"
 )
 
 // this help command gets added to any subcommand that does not define it explicitly
@@ -287,8 +287,8 @@ func bzzd(ctx *cli.Context) error {
 	//setup the ethereum node
 	utils.SetNodeConfig(ctx, &cfg)
 
-	//always disable discovery from p2p package - swarm discovery is done with the `hive` protocol
+	//disable dynamic dialing from p2p/discovery
-	cfg.P2P.NoDiscovery = true
+	cfg.P2P.NoDial = true
 
 	stack, err := node.New(&cfg)
 	if err != nil {
@@ -351,7 +351,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
 func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	//an account is mandatory
 	if bzzaccount == "" {
-		utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT)
+		utils.Fatalf(SwarmErrNoBZZAccount)
 	}
 	// Try to load the arg as a hex key file.
 	if key, err := crypto.LoadECDSA(bzzaccount); err == nil {

cmd/swarm/run_test.go

@@ -82,6 +82,18 @@ func TestMain(m *testing.M) {
 func runSwarm(t *testing.T, args ...string) *cmdtest.TestCmd {
 	tt := cmdtest.NewTestCmd(t, nil)
 
+	found := false
+	for _, v := range args {
+		if v == "--bootnodes" {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		args = append([]string{"--bootnodes", ""}, args...)
+	}
+
 	// Boot "swarm". This actually runs the test binary but the TestMain
 	// function will prevent any tests from running.
 	tt.Run("swarm-test", args...)
@@ -252,6 +264,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 
 	// start the node
 	node.Cmd = runSwarm(t,
+		"--bootnodes", "",
 		"--port", p2pPort,
 		"--nat", "extip:127.0.0.1",
 		"--datadir", dir,
@@ -327,6 +340,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
 
 	// start the node
 	node.Cmd = runSwarm(t,
+		"--bootnodes", "",
 		"--port", p2pPort,
 		"--nat", "extip:127.0.0.1",
 		"--datadir", dir,

cmd/swarm/swarm-smoke/sliding_window.go

@@ -42,23 +42,16 @@ func slidingWindowCmd(ctx *cli.Context, tuid string) error {
 		errc <- slidingWindow(ctx, tuid)
 	}()
 
-	select {
+	err := <-errc
-	case err := <-errc:
 	if err != nil {
 		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
 	}
 	return err
-	case <-time.After(time.Duration(timeout) * time.Second):
-		metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
-
-		return fmt.Errorf("timeout after %v sec", timeout)
-	}
 }
 
 func slidingWindow(ctx *cli.Context, tuid string) error {
 	var hashes []uploadResult //swarm hashes of the uploads
 	nodes := len(hosts)
-	const iterationTimeout = 30 * time.Second
 	log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
 	uploadedBytes := 0
 	networkDepth := 0
@@ -66,6 +59,7 @@ func slidingWindow(ctx *cli.Context, tuid string) error {
 
 outer:
 	for {
+		seed = int(time.Now().UTC().UnixNano())
 		log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
 
 		t1 := time.Now()
@@ -79,6 +73,7 @@ outer:
 		}
 
 		metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
+		metrics.GetOrRegisterGauge("sliding-window.upload-depth", nil).Update(int64(len(hashes)))
 
 		fhash, err := digest(bytes.NewReader(randomBytes))
 		if err != nil {
@@ -90,37 +85,56 @@ outer:
 		hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
 		time.Sleep(time.Duration(syncDelay) * time.Second)
 		uploadedBytes += filesize * 1000
-
+		q := make(chan struct{}, 1)
+		d := make(chan struct{})
+		defer close(q)
+		defer close(d)
 		for i, v := range hashes {
-			timeout := time.After(time.Duration(timeout) * time.Second)
+			timeoutC := time.After(time.Duration(timeout) * time.Second)
 			errored = false
 
-		inner:
+		task:
 			for {
 				select {
-				case <-timeout:
+				case q <- struct{}{}:
-					errored = true
+					go func() {
-					log.Error("error retrieving hash. timeout", "hash idx", i, "err", err)
+						var start time.Time
-					metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
+						done := false
-					break inner
+						for !done {
-				default:
+							log.Info("trying to retrieve hash", "hash", v.hash)
 							idx := 1 + rand.Intn(len(hosts)-1)
 							ruid := uuid.New()[:8]
-					start := time.Now()
+							start = time.Now()
+							// fetch hangs when swarm dies out, so we have to jump through a bit more hoops to actually
+							// catch the timeout, but also allow this retry logic
 							err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "")
 							if err != nil {
-						continue inner
+								log.Error("error fetching hash", "err", err)
+								continue
+							}
+							done = true
 						}
 						metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
-					break inner
+						d <- struct{}{}
+					}()
+				case <-d:
+					<-q
+					break task
+				case <-timeoutC:
+					errored = true
+					log.Error("error retrieving hash. timeout", "hash idx", i)
+					metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
+					break outer
+				default:
 				}
 			}
 
-			if errored {
-				break outer
-			}
 			networkDepth = i
 			metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth))
+			log.Info("sliding window test successfully fetched file", "currentDepth", networkDepth)
+			// this test might take a long time to finish - but we'd like to see metrics while they accumulate and not just when
+			// the test finishes. therefore emit the metrics on each iteration
+			emitMetrics(ctx)
 		}
 	}
 

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -43,7 +44,7 @@ func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
 	errc := make(chan error)
 
 	go func() {
-		errc <- uplaodAndSync(ctx, randomBytes, tuid)
+		errc <- uploadAndSync(ctx, randomBytes, tuid)
 	}()
 
 	select {
@@ -64,44 +65,61 @@ func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
 
 		return e
 	}
+
+	// trigger debug functionality on randomBytes even on successful runs
+	err := trackChunks(randomBytes[:])
+	if err != nil {
+		log.Error(err.Error())
+	}
+
+	return nil
 }
 
 func trackChunks(testData []byte) error {
-	log.Warn("Test timed out; running chunk debug sequence")
+	log.Warn("Test timed out, running chunk debug sequence")
 
 	addrs, err := getAllRefs(testData)
 	if err != nil {
 		return err
 	}
-	log.Trace("All references retrieved")
 
-	// has-chunks
+	for i, ref := range addrs {
+		log.Trace(fmt.Sprintf("ref %d", i), "ref", ref)
+	}
+
 	for _, host := range hosts {
 		httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
-		log.Trace("Calling `Has` on host", "httpHost", httpHost)
+
+		hostChunks := []string{}
+
 		rpcClient, err := rpc.Dial(httpHost)
 		if err != nil {
-			log.Trace("Error dialing host", "err", err)
+			log.Error("Error dialing host", "err", err)
-			return err
+			continue
 		}
-		log.Trace("rpc dial ok")
+
 		var hasInfo []api.HasInfo
 		err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
 		if err != nil {
-			log.Trace("Error calling host", "err", err)
+			log.Error("Error calling host", "err", err)
-			return err
+			continue
 		}
-		log.Trace("rpc call ok")
+
 		count := 0
 		for _, info := range hasInfo {
-			if !info.Has {
+			if info.Has {
+				hostChunks = append(hostChunks, "1")
+			} else {
+				hostChunks = append(hostChunks, "0")
 				count++
-				log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
 			}
 		}
+
 		if count == 0 {
-			log.Info("Host reported to have all chunks", "host", httpHost)
+			log.Info("host reported to have all chunks", "host", host)
 		}
+
+		log.Trace("chunks", "chunks", strings.Join(hostChunks, ""), "host", host)
 	}
 	return nil
 }
@@ -124,7 +142,7 @@ func getAllRefs(testData []byte) (storage.AddressCollection, error) {
 	return fileStore.GetAllReferences(ctx, reader, false)
 }
 
-func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
+func uploadAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
 	log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed)
 
 	t1 := time.Now()

cmd/swarm/swarm-snapshot/create.go

@@ -59,13 +59,16 @@ func createSnapshot(filename string, nodes int, services []string) (err error) {
 	log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
 
 	sim := simulation.New(map[string]simulation.ServiceFunc{
-		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+		"bzz": func(ctx *adapters.ServiceContext, bucket *sync.Map) (node.Service, func(), error) {
 			addr := network.NewAddr(ctx.Config.Node())
 			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
 			hp := network.NewHiveParams()
 			hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
 			hp.Discovery = true // discovery must be enabled when creating a snapshot
 
+			// store the kademlia in the bucket, needed later in the WaitTillHealthy function
+			bucket.Store(simulation.BucketKeyKademlia, kad)
+
 			config := &network.BzzConfig{
 				OverlayAddr:  addr.Over(),
 				UnderlayAddr: addr.Under(),
@@ -76,17 +79,17 @@ func createSnapshot(filename string, nodes int, services []string) (err error) {
 	})
 	defer sim.Close()
 
-	_, err = sim.AddNodes(nodes)
+	ids, err := sim.AddNodes(nodes)
 	if err != nil {
 		return fmt.Errorf("add nodes: %v", err)
 	}
 
-	err = sim.Net.ConnectNodesRing(nil)
+	err = sim.Net.ConnectNodesRing(ids)
 	if err != nil {
 		return fmt.Errorf("connect nodes: %v", err)
 	}
 
-	ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
+	ctx, cancelSimRun := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancelSimRun()
 	if _, err := sim.WaitTillHealthy(ctx); err != nil {
 		return fmt.Errorf("wait for healthy kademlia: %v", err)

cmd/swarm/swarm-snapshot/create_test.go

@@ -48,7 +48,7 @@ func TestSnapshotCreate(t *testing.T) {
 		},
 		{
 			name:  "more nodes",
-			nodes: defaultNodes + 5,
+			nodes: defaultNodes + 4,
 		},
 		{
 			name:     "services",
@@ -81,7 +81,7 @@ func TestSnapshotCreate(t *testing.T) {
 			}
 			testCmd := runSnapshot(t, append(args, file.Name())...)
 
-			testCmd.ExpectExit()
+			testCmd.WaitExit()
 			if code := testCmd.ExitStatus(); code != 0 {
 				t.Fatalf("command exit code %v, expected 0", code)
 			}

cmd/swarm/swarm-snapshot/main.go

@@ -27,7 +27,7 @@ import (
 var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
 
 // default value for "create" command --nodes flag
-const defaultNodes = 10
+const defaultNodes = 8
 
 func main() {
 	err := newApp().Run(os.Args)

cmd/swarm/upload.go

@@ -60,10 +60,10 @@ func upload(ctx *cli.Context) {
 		autoDefaultPath = false
 		file            string
 	)
-	if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
+	if autoDefaultPathString := os.Getenv(SwarmAutoDefaultPath); autoDefaultPathString != "" {
 		b, err := strconv.ParseBool(autoDefaultPathString)
 		if err != nil {
-			utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
+			utils.Fatalf("invalid environment variable %s: %v", SwarmAutoDefaultPath, err)
 		}
 		autoDefaultPath = b
 	}

cmd/utils/cmd.go

@@ -238,7 +238,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
 }
 
 // ImportPreimages imports a batch of exported hash preimages into the database.
-func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
+func ImportPreimages(db ethdb.Database, fn string) error {
 	log.Info("Importing preimages", "file", fn)
 
 	// Open the file handle and potentially unwrap the gzip stream
@@ -285,7 +285,7 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
 
 // ExportPreimages exports all known hash preimages into the specified file,
 // truncating any data already present in the file.
-func ExportPreimages(db *ethdb.LDBDatabase, fn string) error {
+func ExportPreimages(db ethdb.Database, fn string) error {
 	log.Info("Exporting preimages", "file", fn)
 
 	// Open the file handle and potentially wrap with a gzip stream

cmd/utils/flags.go

@@ -37,7 +37,6 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/dashboard"
@@ -199,9 +198,19 @@ var (
 	}
 	LightServFlag = cli.IntFlag{
 		Name:  "lightserv",
-		Usage: "Maximum percentage of time allowed for serving LES requests (0-90)",
+		Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)",
 		Value: 0,
 	}
+	LightBandwidthInFlag = cli.IntFlag{
+		Name:  "lightbwin",
+		Usage: "Incoming bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)",
+		Value: 1000,
+	}
+	LightBandwidthOutFlag = cli.IntFlag{
+		Name:  "lightbwout",
+		Usage: "Outgoing bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)",
+		Value: 5000,
+	}
 	LightPeersFlag = cli.IntFlag{
 		Name:  "lightpeers",
 		Usage: "Maximum number of LES client peers",
@@ -340,11 +349,6 @@ var (
 		Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
 		Value: 25,
 	}
-	TrieCacheGenFlag = cli.IntFlag{
-		Name:  "trie-cache-gens",
-		Usage: "Number of trie node generations to keep in memory",
-		Value: int(state.MaxTrieCacheGen),
-	}
 	// Miner settings
 	MiningEnabledFlag = cli.BoolFlag{
 		Name:  "mine",
@@ -769,6 +773,7 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 			node, err := enode.ParseV4(url)
 			if err != nil {
 				log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
+				continue
 			}
 			cfg.BootstrapNodes = append(cfg.BootstrapNodes, node)
 		}
@@ -796,6 +801,7 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
 
 	cfg.BootstrapNodesV5 = make([]*discv5.Node, 0, len(urls))
 	for _, url := range urls {
+		if url != "" {
 			node, err := discv5.ParseNode(url)
 			if err != nil {
 				log.Error("Bootstrap URL invalid", "enode", url, "err", err)
@@ -803,6 +809,7 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
 			}
 			cfg.BootstrapNodesV5 = append(cfg.BootstrapNodesV5, node)
 		}
+	}
 }
 
 // setListenAddress creates a TCP listening address string from set command
@@ -1305,6 +1312,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(LightServFlag.Name) {
 		cfg.LightServ = ctx.GlobalInt(LightServFlag.Name)
 	}
+	cfg.LightBandwidthIn = ctx.GlobalInt(LightBandwidthInFlag.Name)
+	cfg.LightBandwidthOut = ctx.GlobalInt(LightBandwidthOutFlag.Name)
 	if ctx.GlobalIsSet(LightPeersFlag.Name) {
 		cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name)
 	}
@@ -1420,10 +1429,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 			cfg.MinerGasPrice = big.NewInt(1)
 		}
 	}
-	// TODO(fjl): move trie cache generations into config
-	if gen := ctx.GlobalInt(TrieCacheGenFlag.Name); gen > 0 {
-		state.MaxTrieCacheGen = uint16(gen)
-	}
 }
 
 // SetDashboardConfig applies dashboard related command line flags to the config.
@@ -1536,7 +1541,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
 		name = "lightchaindata"
 	}
-	chainDb, err := stack.OpenDatabase(name, cache, handles)
+	chainDb, err := stack.OpenDatabase(name, cache, handles, "")
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
 	}

common/fdlimit/fdlimit_darwin.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "syscall"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 10240
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	// Retrieve the maximum allowed by dynamic OS limits
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Cap it to OPEN_MAX (10240) because macos is a special snowflake
+	if limit.Max > hardlimit {
+		limit.Max = hardlimit
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_freebsd.go

@@ -43,7 +43,7 @@ func Raise(max uint64) (uint64, error) {
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
 		return 0, err
 	}
-	return limit.Cur, nil
+	return uint64(limit.Cur), nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_unix.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin netbsd openbsd solaris
+// +build linux netbsd openbsd solaris
 
 package fdlimit
 

common/fdlimit/fdlimit_windows.go

@@ -18,6 +18,7 @@ package fdlimit
 
 import "fmt"
 
+// hardlimit is the number of file descriptors allowed at max by the kernel.
 const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process

common/types.go

@@ -202,9 +202,6 @@ func IsHexAddress(s string) bool {
 // Bytes gets the string representation of the underlying address.
 func (a Address) Bytes() []byte { return a[:] }
 
-// Big converts an address to a big integer.
-func (a Address) Big() *big.Int { return new(big.Int).SetBytes(a[:]) }
-
 // Hash converts an address to a hash by left-padding it with zeros.
 func (a Address) Hash() Hash { return BytesToHash(a[:]) }
 

common/types_test.go

@@ -114,8 +114,8 @@ func TestAddressUnmarshalJSON(t *testing.T) {
 			if test.ShouldErr {
 				t.Errorf("test #%d: expected error, got none", i)
 			}
-			if v.Big().Cmp(test.Output) != 0 {
+			if got := new(big.Int).SetBytes(v.Bytes()); got.Cmp(test.Output) != 0 {
-				t.Errorf("test #%d: address mismatch: have %v, want %v", i, v.Big(), test.Output)
+				t.Errorf("test #%d: address mismatch: have %v, want %v", i, got, test.Output)
 			}
 		}
 	}

consensus/clique/snapshot_test.go

@@ -24,10 +24,10 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -246,10 +246,10 @@ func TestClique(t *testing.T) {
 			// Votes from deauthorized signers are discarded immediately (auth votes)
 			signers: []string{"A", "B", "C"},
 			votes: []testerVote{
-				{signer: "C", voted: "B", auth: false},
+				{signer: "C", voted: "D", auth: true},
 				{signer: "A", voted: "C", auth: false},
 				{signer: "B", voted: "C", auth: false},
-				{signer: "A", voted: "B", auth: false},
+				{signer: "A", voted: "D", auth: true},
 			},
 			results: []string{"A", "B"},
 		}, {
@@ -400,7 +400,7 @@ func TestClique(t *testing.T) {
 			copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:])
 		}
 		// Create a pristine blockchain with the genesis injected
-		db := ethdb.NewMemDatabase()
+		db := rawdb.NewMemoryDatabase()
 		genesis.Commit(db)
 
 		// Assemble a chain of headers from the cast votes

containers/docker/develop-alpine/Dockerfile

@@ -1,14 +0,0 @@
-FROM alpine:3.7
-
-RUN \
-  apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
-  git clone --depth 1 https://github.com/ethereum/go-ethereum && \
-  (cd go-ethereum && make geth) && \
-  cp go-ethereum/build/bin/geth /geth && \
-  apk del go git make gcc musl-dev linux-headers && \
-  rm -rf /go-ethereum && rm -rf /var/cache/apk/*
-
-EXPOSE 8545
-EXPOSE 30303
-
-ENTRYPOINT ["/geth"]

containers/docker/develop-ubuntu/Dockerfile

@@ -1,17 +0,0 @@
-FROM ubuntu:xenial
-
-ENV PATH=/usr/lib/go-1.9/bin:$PATH
-
-RUN \
-  apt-get update && apt-get upgrade -q -y && \
-  apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \
-  git clone --depth 1 https://github.com/ethereum/go-ethereum && \
-  (cd go-ethereum && make geth) && \
-  cp go-ethereum/build/bin/geth /geth && \
-  apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \
-  rm -rf /go-ethereum
-
-EXPOSE 8545
-EXPOSE 30303
-
-ENTRYPOINT ["/geth"]

containers/docker/master-alpine/Dockerfile

@@ -1,14 +0,0 @@
-FROM alpine:3.7
-
-RUN \
-  apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
-  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
-  (cd go-ethereum && make geth) && \
-  cp go-ethereum/build/bin/geth /geth && \
-  apk del go git make gcc musl-dev linux-headers && \
-  rm -rf /go-ethereum && rm -rf /var/cache/apk/*
-
-EXPOSE 8545
-EXPOSE 30303
-
-ENTRYPOINT ["/geth"]

containers/docker/master-ubuntu/Dockerfile

@@ -1,17 +0,0 @@
-FROM ubuntu:xenial
-
-ENV PATH=/usr/lib/go-1.9/bin:$PATH
-
-RUN \
-  apt-get update && apt-get upgrade -q -y && \
-  apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \
-  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
-  (cd go-ethereum && make geth) && \
-  cp go-ethereum/build/bin/geth /geth && \
-  apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \
-  rm -rf /go-ethereum
-
-EXPOSE 8545
-EXPOSE 30303
-
-ENTRYPOINT ["/geth"]

core/asm/lex_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func lexAll(src string) []token {
-	ch := Lex("test.asm", []byte(src), false)
+	ch := Lex([]byte(src), false)
 
 	var tokens []token
 	for i := range ch {

core/asm/lexer.go

@@ -95,7 +95,7 @@ type lexer struct {
 
 // lex lexes the program by name with the given source. It returns a
 // channel on which the tokens are delivered.
-func Lex(name string, source []byte, debug bool) <-chan token {
+func Lex(source []byte, debug bool) <-chan token {
 	ch := make(chan token)
 	l := &lexer{
 		input:  string(source),

core/bench_test.go

@@ -150,14 +150,14 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 	// Create the database in memory or in a temporary directory.
 	var db ethdb.Database
 	if !disk {
-		db = ethdb.NewMemDatabase()
+		db = rawdb.NewMemoryDatabase()
 	} else {
 		dir, err := ioutil.TempDir("", "eth-core-bench")
 		if err != nil {
 			b.Fatalf("cannot create temporary directory: %v", err)
 		}
 		defer os.RemoveAll(dir)
-		db, err = ethdb.NewLDBDatabase(dir, 128, 128)
+		db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "")
 		if err != nil {
 			b.Fatalf("cannot create temporary database: %v", err)
 		}
@@ -255,7 +255,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("cannot create temporary directory: %v", err)
 		}
-		db, err := ethdb.NewLDBDatabase(dir, 128, 1024)
+		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
@@ -272,7 +272,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 	}
 	defer os.RemoveAll(dir)
 
-	db, err := ethdb.NewLDBDatabase(dir, 128, 1024)
+	db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
 	if err != nil {
 		b.Fatalf("error opening database at %v: %v", dir, err)
 	}
@@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		db, err := ethdb.NewLDBDatabase(dir, 128, 1024)
+		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}

core/block_validator.go

@@ -77,7 +77,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 // transition, such as amount of used gas, the receipt roots and the state root
 // itself. ValidateState returns a database batch if the validation was a success
 // otherwise nil and an error is returned.
-func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error {
+func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error {
 	header := block.Header()
 	if block.GasUsed() != usedGas {
 		return fmt.Errorf("invalid gas used (remote: %d local: %d)", block.GasUsed(), usedGas)

core/block_validator_test.go

@@ -22,9 +22,9 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -32,7 +32,7 @@ import (
 func TestHeaderVerification(t *testing.T) {
 	// Create a simple chain to verify
 	var (
-		testdb    = ethdb.NewMemDatabase()
+		testdb    = rawdb.NewMemoryDatabase()
 		gspec     = &Genesis{Config: params.TestChainConfig}
 		genesis   = gspec.MustCommit(testdb)
 		blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil)
@@ -84,7 +84,7 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri
 func testHeaderConcurrentVerification(t *testing.T, threads int) {
 	// Create a simple chain to verify
 	var (
-		testdb    = ethdb.NewMemDatabase()
+		testdb    = rawdb.NewMemoryDatabase()
 		gspec     = &Genesis{Config: params.TestChainConfig}
 		genesis   = gspec.MustCommit(testdb)
 		blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil)
@@ -156,7 +156,7 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion
 func testHeaderConcurrentAbortion(t *testing.T, threads int) {
 	// Create a simple chain to verify
 	var (
-		testdb    = ethdb.NewMemDatabase()
+		testdb    = rawdb.NewMemoryDatabase()
 		gspec     = &Genesis{Config: params.TestChainConfig}
 		genesis   = gspec.MustCommit(testdb)
 		blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil)

core/blockchain.go

@@ -43,7 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 var (
@@ -65,7 +65,13 @@ const (
 	triesInMemory       = 128
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
-	BlockChainVersion uint64 = 3
+	//
+	// During the process of upgrading the database version from 3 to 4,
+	// the following incompatible database changes were added.
+	// * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted
+	// * the `Bloom` field of receipt is deleted
+	// * the `BlockIndex` and `TxIndex` fields of txlookup are deleted
+	BlockChainVersion uint64 = 4
 )
 
 // CacheConfig contains the configuration values for the trie caching/pruning
@@ -105,6 +111,7 @@ type BlockChain struct {
 	chainSideFeed event.Feed
 	chainHeadFeed event.Feed
 	logsFeed      event.Feed
+	blockProcFeed event.Feed
 	scope         event.SubscriptionScope
 	genesisBlock  *types.Block
 
@@ -284,7 +291,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	defer bc.chainmu.Unlock()
 
 	// Rewind the header chain, deleting all block bodies until then
-	delFn := func(db rawdb.DatabaseDeleter, hash common.Hash, num uint64) {
+	delFn := func(db ethdb.Deleter, hash common.Hash, num uint64) {
 		rawdb.DeleteBody(db, hash, num)
 	}
 	bc.hc.SetHead(head, delFn)
@@ -335,7 +342,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x…]", hash[:4])
 	}
-	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil {
+	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil {
 		return err
 	}
 	// If all checks out, manually set the head block
@@ -456,7 +463,7 @@ func (bc *BlockChain) repair(head **types.Block) error {
 		if block == nil {
 			return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
 		}
-		(*head) = block
+		*head = block
 	}
 }
 
@@ -1084,15 +1091,25 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 	if len(chain) == 0 {
 		return 0, nil
 	}
+
+	bc.blockProcFeed.Send(true)
+	defer bc.blockProcFeed.Send(false)
+
+	// Remove already known canon-blocks
+	var (
+		block, prev *types.Block
+	)
 	// Do a sanity check that the provided chain is actually ordered and linked
 	for i := 1; i < len(chain); i++ {
-		if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() {
+		block = chain[i]
+		prev = chain[i-1]
+		if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
 			// Chain broke ancestry, log a message (programming error) and skip insertion
-			log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
+			log.Error("Non contiguous block insert", "number", block.Number(), "hash", block.Hash(),
-				"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
+				"parent", block.ParentHash(), "prevnumber", prev.Number(), "prevhash", prev.Hash())
 
-			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, prev.NumberU64(),
-				chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
+				prev.Hash().Bytes()[:4], i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
 		}
 	}
 	// Pre-checks passed, start the full block imports
@@ -1146,6 +1163,22 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 	it := newInsertIterator(chain, results, bc.Validator())
 
 	block, err := it.next()
+
+	// Left-trim all the known blocks
+	if err == ErrKnownBlock {
+		// First block (and state) is known
+		//   1. We did a roll-back, and should now do a re-import
+		//   2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
+		// 	    from the canonical chain, which has not been verified.
+		// Skip all known blocks that are behind us
+		current := bc.CurrentBlock().NumberU64()
+		for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
+			stats.ignored++
+			block, err = it.next()
+		}
+		// Falls through to the block import
+	}
+
 	switch {
 	// First block is pruned, insert as sidechain and reorg only if TD grows enough
 	case err == consensus.ErrPrunedAncestor:
@@ -1165,20 +1198,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 		// If there are any still remaining, mark as ignored
 		return it.index, events, coalescedLogs, err
 
-	// First block (and state) is known
-	//   1. We did a roll-back, and should now do a re-import
-	//   2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
-	// 	    from the canonical chain, which has not been verified.
-	case err == ErrKnownBlock:
-		// Skip all known blocks that behind us
-		current := bc.CurrentBlock().NumberU64()
-
-		for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
-			stats.ignored++
-			block, err = it.next()
-		}
-		// Falls through to the block import
-
 	// Some other error occurred, abort
 	case err != nil:
 		stats.ignored += len(it.chain)
@@ -1202,9 +1221,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 
 		parent := it.previous()
 		if parent == nil {
-			parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
+			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
-		state, err := state.New(parent.Root(), bc.stateCache)
+		state, err := state.New(parent.Root, bc.stateCache)
 		if err != nil {
 			return it.index, events, coalescedLogs, err
 		}
@@ -1217,7 +1236,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
 			return it.index, events, coalescedLogs, err
 		}
 		// Validate the state using the default validator
-		if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
+		if err := bc.Validator().ValidateState(block, state, receipts, usedGas); err != nil {
 			bc.reportBlock(block, receipts, err)
 			return it.index, events, coalescedLogs, err
 		}
@@ -1304,7 +1323,12 @@ func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (i
 	for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
 		// Check the canonical state root for that number
 		if number := block.NumberU64(); current.NumberU64() >= number {
-			if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
+			canonical := bc.GetBlockByNumber(number)
+			if canonical != nil && canonical.Hash() == block.Hash() {
+				// Not a sidechain block, this is a re-import of a canon block which has it's state pruned
+				continue
+			}
+			if canonical != nil && canonical.Root() == block.Root() {
 				// This is most likely a shadow-state attack. When a fork is imported into the
 				// database, and it eventually reaches a block height which is not pruned, we
 				// just found that the state already exist! This means that the sidechain block
@@ -1344,7 +1368,7 @@ func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (i
 	// blocks to regenerate the required state
 	localTd := bc.GetTd(current.Hash(), current.NumberU64())
 	if localTd.Cmp(externTd) > 0 {
-		log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
+		log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().Number, "sidetd", externTd, "localtd", localTd)
 		return it.index, nil, nil, err
 	}
 	// Gather all the sidechain hashes (full blocks may be memory heavy)
@@ -1352,7 +1376,7 @@ func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (i
 		hashes  []common.Hash
 		numbers []uint64
 	)
-	parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
+	parent := it.previous()
 	for parent != nil && !bc.HasState(parent.Root) {
 		hashes = append(hashes, parent.Hash())
 		numbers = append(numbers, parent.Number.Uint64())
@@ -1706,3 +1730,9 @@ func (bc *BlockChain) SubscribeChainSideEvent(ch chan<- ChainSideEvent) event.Su
 func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
 	return bc.scope.Track(bc.logsFeed.Subscribe(ch))
 }
+
+// SubscribeBlockProcessingEvent registers a subscription of bool where true means
+// block processing has started while false means it has stopped.
+func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription {
+	return bc.scope.Track(bc.blockProcFeed.Subscribe(ch))
+}

core/blockchain_insert.go

@@ -111,12 +111,12 @@ func (it *insertIterator) next() (*types.Block, error) {
 	return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
 }
 
-// previous returns the previous block was being processed, or nil
+// previous returns the previous header that was being processed, or nil.
-func (it *insertIterator) previous() *types.Block {
+func (it *insertIterator) previous() *types.Header {
 	if it.index < 1 {
 		return nil
 	}
-	return it.chain[it.index-1]
+	return it.chain[it.index-1].Header()
 }
 
 // first returns the first block in the it.

core/blockchain_test.go

@@ -46,7 +46,7 @@ var (
 // header only chain.
 func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) {
 	var (
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 		genesis = new(Genesis).MustCommit(db)
 	)
 
@@ -149,7 +149,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 			blockchain.reportBlock(block, receipts, err)
 			return err
 		}
-		err = blockchain.validator.ValidateState(block, blockchain.GetBlockByHash(block.ParentHash()), statedb, receipts, usedGas)
+		err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas)
 		if err != nil {
 			blockchain.reportBlock(block, receipts, err)
 			return err
@@ -586,7 +586,7 @@ func testInsertNonceError(t *testing.T, full bool) {
 func TestFastVsFullChains(t *testing.T) {
 	// Configure and generate a sample block chain
 	var (
-		gendb   = ethdb.NewMemDatabase()
+		gendb   = rawdb.NewMemoryDatabase()
 		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		address = crypto.PubkeyToAddress(key.PublicKey)
 		funds   = big.NewInt(1000000000)
@@ -616,7 +616,7 @@ func TestFastVsFullChains(t *testing.T) {
 		}
 	})
 	// Import the chain as an archive node for the comparison baseline
-	archiveDb := ethdb.NewMemDatabase()
+	archiveDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(archiveDb)
 	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer archive.Stop()
@@ -625,7 +625,7 @@ func TestFastVsFullChains(t *testing.T) {
 		t.Fatalf("failed to process block %d: %v", n, err)
 	}
 	// Fast import the chain as a non-archive node to test
-	fastDb := ethdb.NewMemDatabase()
+	fastDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(fastDb)
 	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
@@ -674,7 +674,7 @@ func TestFastVsFullChains(t *testing.T) {
 func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Configure and generate a sample block chain
 	var (
-		gendb   = ethdb.NewMemDatabase()
+		gendb   = rawdb.NewMemoryDatabase()
 		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		address = crypto.PubkeyToAddress(key.PublicKey)
 		funds   = big.NewInt(1000000000)
@@ -702,7 +702,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 		}
 	}
 	// Import the chain as an archive node and ensure all pointers are updated
-	archiveDb := ethdb.NewMemDatabase()
+	archiveDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(archiveDb)
 
 	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
@@ -716,7 +716,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	assert(t, "archive", archive, height/2, height/2, height/2)
 
 	// Import the chain as a non-archive node and ensure all pointers are updated
-	fastDb := ethdb.NewMemDatabase()
+	fastDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(fastDb)
 	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
@@ -736,7 +736,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	assert(t, "fast", fast, height/2, height/2, 0)
 
 	// Import the chain as a light node and ensure all pointers are updated
-	lightDb := ethdb.NewMemDatabase()
+	lightDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(lightDb)
 
 	light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
@@ -759,7 +759,7 @@ func TestChainTxReorgs(t *testing.T) {
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
 		addr2   = crypto.PubkeyToAddress(key2.PublicKey)
 		addr3   = crypto.PubkeyToAddress(key3.PublicKey)
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 		gspec   = &Genesis{
 			Config:   params.TestChainConfig,
 			GasLimit: 3141592,
@@ -871,7 +871,7 @@ func TestLogReorgs(t *testing.T) {
 	var (
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 		// this code generates a log
 		code    = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00")
 		gspec   = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}}
@@ -915,7 +915,7 @@ func TestLogReorgs(t *testing.T) {
 
 func TestReorgSideEvent(t *testing.T) {
 	var (
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
 		gspec   = &Genesis{
@@ -1043,7 +1043,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 func TestEIP155Transition(t *testing.T) {
 	// Configure and generate a sample block chain
 	var (
-		db         = ethdb.NewMemDatabase()
+		db         = rawdb.NewMemoryDatabase()
 		key, _     = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		address    = crypto.PubkeyToAddress(key.PublicKey)
 		funds      = big.NewInt(1000000000)
@@ -1146,7 +1146,7 @@ func TestEIP155Transition(t *testing.T) {
 func TestEIP161AccountRemoval(t *testing.T) {
 	// Configure and generate a sample block chain
 	var (
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		address = crypto.PubkeyToAddress(key.PublicKey)
 		funds   = big.NewInt(1000000000)
@@ -1218,7 +1218,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
 	// Generate a canonical chain to act as the main dataset
 	engine := ethash.NewFaker()
 
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	genesis := new(Genesis).MustCommit(db)
 	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
 
@@ -1234,7 +1234,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
 	}
 	// Import the canonical and fork chain side by side, verifying the current block
 	// and current header consistency
-	diskdb := ethdb.NewMemDatabase()
+	diskdb := rawdb.NewMemoryDatabase()
 	new(Genesis).MustCommit(diskdb)
 
 	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
@@ -1263,7 +1263,7 @@ func TestTrieForkGC(t *testing.T) {
 	// Generate a canonical chain to act as the main dataset
 	engine := ethash.NewFaker()
 
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	genesis := new(Genesis).MustCommit(db)
 	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
 
@@ -1278,7 +1278,7 @@ func TestTrieForkGC(t *testing.T) {
 		forks[i] = fork[0]
 	}
 	// Import the canonical and fork chain side by side, forcing the trie cache to cache both
-	diskdb := ethdb.NewMemDatabase()
+	diskdb := rawdb.NewMemoryDatabase()
 	new(Genesis).MustCommit(diskdb)
 
 	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
@@ -1309,7 +1309,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 	// Generate the original common chain segment and the two competing forks
 	engine := ethash.NewFaker()
 
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	genesis := new(Genesis).MustCommit(db)
 
 	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
@@ -1317,7 +1317,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 	competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
 
 	// Import the shared chain and the original canonical one
-	diskdb := ethdb.NewMemDatabase()
+	diskdb := rawdb.NewMemoryDatabase()
 	new(Genesis).MustCommit(diskdb)
 
 	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
@@ -1377,7 +1377,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
 	)
 	// Generate the original common chain segment and the two competing forks
 	engine := ethash.NewFaker()
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	genesis := gspec.MustCommit(db)
 
 	blockGenerator := func(i int, block *BlockGen) {
@@ -1399,7 +1399,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// Import the shared chain and the original canonical one
-		diskdb := ethdb.NewMemDatabase()
+		diskdb := rawdb.NewMemoryDatabase()
 		gspec.MustCommit(diskdb)
 
 		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
@@ -1412,7 +1412,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
 		}
 		b.StopTimer()
 		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
-			b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
+			b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got)
 
 		}
 	}
@@ -1477,7 +1477,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
 func TestLowDiffLongChain(t *testing.T) {
 	// Generate a canonical chain to act as the main dataset
 	engine := ethash.NewFaker()
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	genesis := new(Genesis).MustCommit(db)
 
 	// We must use a pretty long chain to ensure that the fork doesn't overtake us
@@ -1488,7 +1488,7 @@ func TestLowDiffLongChain(t *testing.T) {
 	})
 
 	// Import the canonical chain
-	diskdb := ethdb.NewMemDatabase()
+	diskdb := rawdb.NewMemoryDatabase()
 	new(Genesis).MustCommit(diskdb)
 
 	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
@@ -1521,3 +1521,85 @@ func TestLowDiffLongChain(t *testing.T) {
 		header = chain.GetHeader(header.ParentHash, number-1)
 	}
 }
+
+// Tests that importing a sidechain (S), where
+// - S is sidechain, containing blocks [Sn...Sm]
+// - C is canon chain, containing blocks [G..Cn..Cm]
+// - A common ancestor is placed at prune-point + blocksBetweenCommonAncestorAndPruneblock
+// - The sidechain S is prepended with numCanonBlocksInSidechain blocks from the canon chain
+func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int) {
+
+	// Generate a canonical chain to act as the main dataset
+	engine := ethash.NewFaker()
+	db := rawdb.NewMemoryDatabase()
+	genesis := new(Genesis).MustCommit(db)
+
+	// Generate and import the canonical chain
+	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, nil)
+	diskdb := rawdb.NewMemoryDatabase()
+	new(Genesis).MustCommit(diskdb)
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	if n, err := chain.InsertChain(blocks); err != nil {
+		t.Fatalf("block %d: failed to insert into chain: %v", n, err)
+	}
+
+	lastPrunedIndex := len(blocks) - triesInMemory - 1
+	lastPrunedBlock := blocks[lastPrunedIndex]
+	firstNonPrunedBlock := blocks[len(blocks)-triesInMemory]
+
+	// Verify pruning of lastPrunedBlock
+	if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) {
+		t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64())
+	}
+	// Verify firstNonPrunedBlock is not pruned
+	if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) {
+		t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64())
+	}
+	// Generate the sidechain
+	// First block should be a known block, block after should be a pruned block. So
+	// canon(pruned), side, side...
+
+	// Generate fork chain, make it longer than canon
+	parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock
+	parent := blocks[parentIndex]
+	fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 2*triesInMemory, func(i int, b *BlockGen) {
+		b.SetCoinbase(common.Address{2})
+	})
+	// Prepend the parent(s)
+	var sidechain []*types.Block
+	for i := numCanonBlocksInSidechain; i > 0; i-- {
+		sidechain = append(sidechain, blocks[parentIndex+1-i])
+	}
+	sidechain = append(sidechain, fork...)
+	_, err = chain.InsertChain(sidechain)
+	if err != nil {
+		t.Errorf("Got error, %v", err)
+	}
+	head := chain.CurrentBlock()
+	if got := fork[len(fork)-1].Hash(); got != head.Hash() {
+		t.Fatalf("head wrong, expected %x got %x", head.Hash(), got)
+	}
+}
+
+// Tests that importing a sidechain (S), where
+// - S is sidechain, containing blocks [Sn...Sm]
+// - C is canon chain, containing blocks [G..Cn..Cm]
+// - The common ancestor Cc is pruned
+// - The first block in S: Sn, is == Cn
+// That is: the sidechain for import contains some blocks already present in canon chain.
+// So the blocks are
+// [ Cn, Cn+1, Cc, Sn+3 ... Sm]
+//   ^    ^    ^  pruned
+func TestPrunedImportSide(t *testing.T) {
+	//glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false)))
+	//glogger.Verbosity(3)
+	//log.Root().SetHandler(log.Handler(glogger))
+	testSideImport(t, 3, 3)
+	testSideImport(t, 3, -3)
+	testSideImport(t, 10, 0)
+	testSideImport(t, 1, 10)
+	testSideImport(t, 1, -10)
+}

core/chain_indexer.go

@@ -97,7 +97,7 @@ type ChainIndexer struct {
 // NewChainIndexer creates a new chain indexer to do background processing on
 // chain segments of a given size after certain number of confirmations passed.
 // The throttling parameter might be used to prevent database thrashing.
-func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
+func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
 	c := &ChainIndexer{
 		chainDb:     chainDb,
 		indexDb:     indexDb,

core/chain_indexer_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // Runs multiple tests with randomized parameters.
@@ -49,7 +48,7 @@ func TestChainIndexerWithChildren(t *testing.T) {
 // multiple backends. The section size and required confirmation count parameters
 // are randomized.
 func testChainIndexer(t *testing.T, count int) {
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	defer db.Close()
 
 	// Create a chain of indexers and ensure they all report empty
@@ -60,7 +59,7 @@ func testChainIndexer(t *testing.T, count int) {
 			confirmsReq = uint64(rand.Intn(10))
 		)
 		backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)}
-		backends[i].indexer = NewChainIndexer(db, ethdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i))
+		backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i))
 
 		if sections, _, _ := backends[i].indexer.Sections(); sections != 0 {
 			t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0)

core/chain_makers_test.go

@@ -21,10 +21,10 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -36,7 +36,7 @@ func ExampleGenerateChain() {
 		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
 		addr2   = crypto.PubkeyToAddress(key2.PublicKey)
 		addr3   = crypto.PubkeyToAddress(key3.PublicKey)
-		db      = ethdb.NewMemDatabase()
+		db      = rawdb.NewMemoryDatabase()
 	)
 
 	// Ensure that key1 has some funds in the genesis block.

core/dao_test.go

@@ -21,8 +21,8 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -32,13 +32,13 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	forkBlock := big.NewInt(32)
 
 	// Generate a common prefix for both pro-forkers and non-forkers
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	gspec := new(Genesis)
 	genesis := gspec.MustCommit(db)
 	prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {})
 
 	// Create the concurrent, conflicting two nodes
-	proDb := ethdb.NewMemDatabase()
+	proDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(proDb)
 
 	proConf := *params.TestChainConfig
@@ -48,7 +48,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer proBc.Stop()
 
-	conDb := ethdb.NewMemDatabase()
+	conDb := rawdb.NewMemoryDatabase()
 	gspec.MustCommit(conDb)
 
 	conConf := *params.TestChainConfig
@@ -67,7 +67,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks
 	for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ {
 		// Create a pro-fork block, and try to feed into the no-fork chain
-		db = ethdb.NewMemDatabase()
+		db = rawdb.NewMemoryDatabase()
 		gspec.MustCommit(db)
 		bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 		defer bc.Stop()
@@ -92,7 +92,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 			t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err)
 		}
 		// Create a no-fork block, and try to feed into the pro-fork chain
-		db = ethdb.NewMemDatabase()
+		db = rawdb.NewMemoryDatabase()
 		gspec.MustCommit(db)
 		bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 		defer bc.Stop()
@@ -118,7 +118,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		}
 	}
 	// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
-	db = ethdb.NewMemDatabase()
+	db = rawdb.NewMemoryDatabase()
 	gspec.MustCommit(db)
 	bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer bc.Stop()
@@ -138,7 +138,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err)
 	}
 	// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
-	db = ethdb.NewMemDatabase()
+	db = rawdb.NewMemoryDatabase()
 	gspec.MustCommit(db)
 	bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer bc.Stop()

core/genesis.go

@@ -134,7 +134,7 @@ type GenesisMismatchError struct {
 }
 
 func (e *GenesisMismatchError) Error() string {
-	return fmt.Sprintf("database already contains an incompatible genesis block (have %x, new %x)", e.Stored[:8], e.New[:8])
+	return fmt.Sprintf("database contains incompatible genesis (have %x, new %x)", e.Stored, e.New)
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.
@@ -228,7 +228,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 // to the given database (or discards it if nil).
 func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
 	if db == nil {
-		db = ethdb.NewMemDatabase()
+		db = rawdb.NewMemoryDatabase()
 	}
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
 	for addr, account := range g.Alloc {

core/genesis_test.go

@@ -141,7 +141,7 @@ func TestSetupGenesis(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		db := ethdb.NewMemDatabase()
+		db := rawdb.NewMemoryDatabase()
 		config, hash, err := test.fn(db)
 		// Check the return values.
 		if !reflect.DeepEqual(err, test.wantErr) {

core/headerchain.go

@@ -33,7 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 const (
@@ -455,7 +455,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
 
 // DeleteCallback is a callback function that is called by SetHead before
 // each header is deleted.
-type DeleteCallback func(rawdb.DatabaseDeleter, common.Hash, uint64)
+type DeleteCallback func(ethdb.Deleter, common.Hash, uint64)
 
 // SetHead rewinds the local chain to a new head. Everything above the new head
 // will be deleted and the new one set.

core/helper_test.go

@@ -19,6 +19,7 @@ package core
 import (
 	"container/list"
 
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -78,7 +79,7 @@ func (tm *TestManager) Db() ethdb.Database {
 func NewTestManager() *TestManager {
 	testManager := &TestManager{}
 	testManager.eventMux = new(event.TypeMux)
-	testManager.db = ethdb.NewMemDatabase()
+	testManager.db = rawdb.NewMemoryDatabase()
 	// testManager.txPool = NewTxPool(testManager)
 	// testManager.blockChain = NewBlockChain(testManager)
 	// testManager.stateManager = NewStateManager(testManager)

core/mkalloc.go

@@ -52,7 +52,8 @@ func makelist(g *core.Genesis) allocList {
 		if len(account.Storage) > 0 || len(account.Code) > 0 || account.Nonce != 0 {
 			panic(fmt.Sprintf("can't encode account %x", addr))
 		}
-		a = append(a, allocItem{addr.Big(), account.Balance})
+		bigAddr := new(big.Int).SetBytes(addr.Bytes())
+		a = append(a, allocItem{bigAddr, account.Balance})
 	}
 	sort.Sort(a)
 	return a

core/rawdb/accessors_chain.go

@@ -23,12 +23,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // ReadCanonicalHash retrieves the hash assigned to a canonical block number.
-func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash {
+func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
 	data, _ := db.Get(headerHashKey(number))
 	if len(data) == 0 {
 		return common.Hash{}
@@ -37,21 +38,21 @@ func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash {
 }
 
 // WriteCanonicalHash stores the hash assigned to a canonical block number.
-func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) {
+func WriteCanonicalHash(db ethdb.Writer, hash common.Hash, number uint64) {
 	if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil {
 		log.Crit("Failed to store number to hash mapping", "err", err)
 	}
 }
 
 // DeleteCanonicalHash removes the number to hash canonical mapping.
-func DeleteCanonicalHash(db DatabaseDeleter, number uint64) {
+func DeleteCanonicalHash(db ethdb.Deleter, number uint64) {
 	if err := db.Delete(headerHashKey(number)); err != nil {
 		log.Crit("Failed to delete number to hash mapping", "err", err)
 	}
 }
 
 // ReadHeaderNumber returns the header number assigned to a hash.
-func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 {
+func ReadHeaderNumber(db ethdb.Reader, hash common.Hash) *uint64 {
 	data, _ := db.Get(headerNumberKey(hash))
 	if len(data) != 8 {
 		return nil
@@ -61,7 +62,7 @@ func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 {
 }
 
 // ReadHeadHeaderHash retrieves the hash of the current canonical head header.
-func ReadHeadHeaderHash(db DatabaseReader) common.Hash {
+func ReadHeadHeaderHash(db ethdb.Reader) common.Hash {
 	data, _ := db.Get(headHeaderKey)
 	if len(data) == 0 {
 		return common.Hash{}
@@ -70,14 +71,14 @@ func ReadHeadHeaderHash(db DatabaseReader) common.Hash {
 }
 
 // WriteHeadHeaderHash stores the hash of the current canonical head header.
-func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) {
+func WriteHeadHeaderHash(db ethdb.Writer, hash common.Hash) {
 	if err := db.Put(headHeaderKey, hash.Bytes()); err != nil {
 		log.Crit("Failed to store last header's hash", "err", err)
 	}
 }
 
 // ReadHeadBlockHash retrieves the hash of the current canonical head block.
-func ReadHeadBlockHash(db DatabaseReader) common.Hash {
+func ReadHeadBlockHash(db ethdb.Reader) common.Hash {
 	data, _ := db.Get(headBlockKey)
 	if len(data) == 0 {
 		return common.Hash{}
@@ -86,14 +87,14 @@ func ReadHeadBlockHash(db DatabaseReader) common.Hash {
 }
 
 // WriteHeadBlockHash stores the head block's hash.
-func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) {
+func WriteHeadBlockHash(db ethdb.Writer, hash common.Hash) {
 	if err := db.Put(headBlockKey, hash.Bytes()); err != nil {
 		log.Crit("Failed to store last block's hash", "err", err)
 	}
 }
 
 // ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block.
-func ReadHeadFastBlockHash(db DatabaseReader) common.Hash {
+func ReadHeadFastBlockHash(db ethdb.Reader) common.Hash {
 	data, _ := db.Get(headFastBlockKey)
 	if len(data) == 0 {
 		return common.Hash{}
@@ -102,7 +103,7 @@ func ReadHeadFastBlockHash(db DatabaseReader) common.Hash {
 }
 
 // WriteHeadFastBlockHash stores the hash of the current fast-sync head block.
-func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) {
+func WriteHeadFastBlockHash(db ethdb.Writer, hash common.Hash) {
 	if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil {
 		log.Crit("Failed to store last fast block's hash", "err", err)
 	}
@@ -110,7 +111,7 @@ func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) {
 
 // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow
 // reporting correct numbers across restarts.
-func ReadFastTrieProgress(db DatabaseReader) uint64 {
+func ReadFastTrieProgress(db ethdb.Reader) uint64 {
 	data, _ := db.Get(fastTrieProgressKey)
 	if len(data) == 0 {
 		return 0
@@ -120,20 +121,20 @@ func ReadFastTrieProgress(db DatabaseReader) uint64 {
 
 // WriteFastTrieProgress stores the fast sync trie process counter to support
 // retrieving it across restarts.
-func WriteFastTrieProgress(db DatabaseWriter, count uint64) {
+func WriteFastTrieProgress(db ethdb.Writer, count uint64) {
 	if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil {
 		log.Crit("Failed to store fast sync trie progress", "err", err)
 	}
 }
 
 // ReadHeaderRLP retrieves a block header in its raw RLP database encoding.
-func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue {
+func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	data, _ := db.Get(headerKey(number, hash))
 	return data
 }
 
 // HasHeader verifies the existence of a block header corresponding to the hash.
-func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool {
+func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool {
 	if has, err := db.Has(headerKey(number, hash)); !has || err != nil {
 		return false
 	}
@@ -141,7 +142,7 @@ func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool {
 }
 
 // ReadHeader retrieves the block header corresponding to the hash.
-func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header {
+func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header {
 	data := ReadHeaderRLP(db, hash, number)
 	if len(data) == 0 {
 		return nil
@@ -156,7 +157,7 @@ func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Heade
 
 // WriteHeader stores a block header into the database and also stores the hash-
 // to-number mapping.
-func WriteHeader(db DatabaseWriter, header *types.Header) {
+func WriteHeader(db ethdb.Writer, header *types.Header) {
 	// Write the hash -> number mapping
 	var (
 		hash    = header.Hash()
@@ -179,30 +180,36 @@ func WriteHeader(db DatabaseWriter, header *types.Header) {
 }
 
 // DeleteHeader removes all block header data associated with a hash.
-func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) {
+func DeleteHeader(db ethdb.Deleter, hash common.Hash, number uint64) {
-	if err := db.Delete(headerKey(number, hash)); err != nil {
+	deleteHeaderWithoutNumber(db, hash, number)
-		log.Crit("Failed to delete header", "err", err)
-	}
 	if err := db.Delete(headerNumberKey(hash)); err != nil {
 		log.Crit("Failed to delete hash to number mapping", "err", err)
 	}
 }
 
+// deleteHeaderWithoutNumber removes only the block header but does not remove
+// the hash to number mapping.
+func deleteHeaderWithoutNumber(db ethdb.Deleter, hash common.Hash, number uint64) {
+	if err := db.Delete(headerKey(number, hash)); err != nil {
+		log.Crit("Failed to delete header", "err", err)
+	}
+}
+
 // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding.
-func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue {
+func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	data, _ := db.Get(blockBodyKey(number, hash))
 	return data
 }
 
 // WriteBodyRLP stores an RLP encoded block body into the database.
-func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) {
+func WriteBodyRLP(db ethdb.Writer, hash common.Hash, number uint64, rlp rlp.RawValue) {
 	if err := db.Put(blockBodyKey(number, hash), rlp); err != nil {
 		log.Crit("Failed to store block body", "err", err)
 	}
 }
 
 // HasBody verifies the existence of a block body corresponding to the hash.
-func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool {
+func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool {
 	if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil {
 		return false
 	}
@@ -210,7 +217,7 @@ func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool {
 }
 
 // ReadBody retrieves the block body corresponding to the hash.
-func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body {
+func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body {
 	data := ReadBodyRLP(db, hash, number)
 	if len(data) == 0 {
 		return nil
@@ -224,7 +231,7 @@ func ReadBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body {
 }
 
 // WriteBody storea a block body into the database.
-func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) {
+func WriteBody(db ethdb.Writer, hash common.Hash, number uint64, body *types.Body) {
 	data, err := rlp.EncodeToBytes(body)
 	if err != nil {
 		log.Crit("Failed to RLP encode body", "err", err)
@@ -233,15 +240,21 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B
 }
 
 // DeleteBody removes all block body data associated with a hash.
-func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) {
+func DeleteBody(db ethdb.Deleter, hash common.Hash, number uint64) {
 	if err := db.Delete(blockBodyKey(number, hash)); err != nil {
 		log.Crit("Failed to delete block body", "err", err)
 	}
 }
 
-// ReadTd retrieves a block's total difficulty corresponding to the hash.
+// ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding.
-func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int {
+func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	data, _ := db.Get(headerTDKey(number, hash))
+	return data
+}
+
+// ReadTd retrieves a block's total difficulty corresponding to the hash.
+func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int {
+	data := ReadTdRLP(db, hash, number)
 	if len(data) == 0 {
 		return nil
 	}
@@ -254,7 +267,7 @@ func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int {
 }
 
 // WriteTd stores the total difficulty of a block into the database.
-func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) {
+func WriteTd(db ethdb.Writer, hash common.Hash, number uint64, td *big.Int) {
 	data, err := rlp.EncodeToBytes(td)
 	if err != nil {
 		log.Crit("Failed to RLP encode block total difficulty", "err", err)
@@ -265,7 +278,7 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) {
 }
 
 // DeleteTd removes all block total difficulty data associated with a hash.
-func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
+func DeleteTd(db ethdb.Deleter, hash common.Hash, number uint64) {
 	if err := db.Delete(headerTDKey(number, hash)); err != nil {
 		log.Crit("Failed to delete block total difficulty", "err", err)
 	}
@@ -273,17 +286,23 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
 
 // HasReceipts verifies the existence of all the transaction receipts belonging
 // to a block.
-func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool {
 	if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
 		return false
 	}
 	return true
 }
 
-// ReadReceipts retrieves all the transaction receipts belonging to a block.
+// ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding.
-func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
+func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
-	// Retrieve the flattened receipt slice
 	data, _ := db.Get(blockReceiptsKey(number, hash))
+	return data
+}
+
+// ReadReceipts retrieves all the transaction receipts belonging to a block.
+func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64) types.Receipts {
+	// Retrieve the flattened receipt slice
+	data := ReadReceiptsRLP(db, hash, number)
 	if len(data) == 0 {
 		return nil
 	}
@@ -294,14 +313,24 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece
 		return nil
 	}
 	receipts := make(types.Receipts, len(storageReceipts))
+	logIndex := uint(0)
 	for i, receipt := range storageReceipts {
+		// Assemble deriving fields for log.
+		for _, log := range receipt.Logs {
+			log.TxHash = receipt.TxHash
+			log.BlockHash = hash
+			log.BlockNumber = number
+			log.TxIndex = uint(i)
+			log.Index = logIndex
+			logIndex += 1
+		}
 		receipts[i] = (*types.Receipt)(receipt)
 	}
 	return receipts
 }
 
 // WriteReceipts stores all the transaction receipts belonging to a block.
-func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) {
+func WriteReceipts(db ethdb.Writer, hash common.Hash, number uint64, receipts types.Receipts) {
 	// Convert the receipts into their storage form and serialize them
 	storageReceipts := make([]*types.ReceiptForStorage, len(receipts))
 	for i, receipt := range receipts {
@@ -318,7 +347,7 @@ func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts
 }
 
 // DeleteReceipts removes all receipt data associated with a block hash.
-func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) {
+func DeleteReceipts(db ethdb.Deleter, hash common.Hash, number uint64) {
 	if err := db.Delete(blockReceiptsKey(number, hash)); err != nil {
 		log.Crit("Failed to delete block receipts", "err", err)
 	}
@@ -330,7 +359,7 @@ func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) {
 //
 // Note, due to concurrent download of header and block body the header and thus
 // canonical hash can be stored in the database but the body data not (yet).
-func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block {
+func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block {
 	header := ReadHeader(db, hash, number)
 	if header == nil {
 		return nil
@@ -343,21 +372,30 @@ func ReadBlock(db DatabaseReader, hash common.Hash, number uint64) *types.Block
 }
 
 // WriteBlock serializes a block into the database, header and body separately.
-func WriteBlock(db DatabaseWriter, block *types.Block) {
+func WriteBlock(db ethdb.Writer, block *types.Block) {
 	WriteBody(db, block.Hash(), block.NumberU64(), block.Body())
 	WriteHeader(db, block.Header())
 }
 
 // DeleteBlock removes all block data associated with a hash.
-func DeleteBlock(db DatabaseDeleter, hash common.Hash, number uint64) {
+func DeleteBlock(db ethdb.Deleter, hash common.Hash, number uint64) {
 	DeleteReceipts(db, hash, number)
 	DeleteHeader(db, hash, number)
 	DeleteBody(db, hash, number)
 	DeleteTd(db, hash, number)
 }
 
+// deleteBlockWithoutNumber removes all block data associated with a hash, except
+// the hash to number mapping.
+func deleteBlockWithoutNumber(db ethdb.Deleter, hash common.Hash, number uint64) {
+	DeleteReceipts(db, hash, number)
+	deleteHeaderWithoutNumber(db, hash, number)
+	DeleteBody(db, hash, number)
+	DeleteTd(db, hash, number)
+}
+
 // FindCommonAncestor returns the last common ancestor of two block headers
-func FindCommonAncestor(db DatabaseReader, a, b *types.Header) *types.Header {
+func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header {
 	for bn := b.Number.Uint64(); a.Number.Uint64() > bn; {
 		a = ReadHeader(db, a.ParentHash, a.Number.Uint64()-1)
 		if a == nil {

core/rawdb/accessors_chain_test.go

@@ -23,14 +23,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/crypto/sha3"
 )
 
 // Tests block header storage and retrieval operations.
 func TestHeaderStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	// Create a test header to move around the database and make sure it's really new
 	header := &types.Header{Number: big.NewInt(42), Extra: []byte("test header")}
@@ -63,7 +62,7 @@ func TestHeaderStorage(t *testing.T) {
 
 // Tests block body storage and retrieval operations.
 func TestBodyStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	// Create a test body to move around the database and make sure it's really new
 	body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}}
@@ -101,7 +100,7 @@ func TestBodyStorage(t *testing.T) {
 
 // Tests block storage and retrieval operations.
 func TestBlockStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	// Create a test block to move around the database and make sure it's really new
 	block := types.NewBlockWithHeader(&types.Header{
@@ -151,7 +150,7 @@ func TestBlockStorage(t *testing.T) {
 
 // Tests that partial block contents don't get reassembled into full blocks.
 func TestPartialBlockStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 	block := types.NewBlockWithHeader(&types.Header{
 		Extra:       []byte("test block"),
 		UncleHash:   types.EmptyUncleHash,
@@ -185,7 +184,7 @@ func TestPartialBlockStorage(t *testing.T) {
 
 // Tests block total difficulty storage and retrieval operations.
 func TestTdStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	// Create a test TD to move around the database and make sure it's really new
 	hash, td := common.Hash{}, big.NewInt(314)
@@ -208,7 +207,7 @@ func TestTdStorage(t *testing.T) {
 
 // Tests that canonical numbers can be mapped to hashes and retrieved.
 func TestCanonicalMappingStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	// Create a test canonical number and assinged hash to move around
 	hash, number := common.Hash{0: 0xff}, uint64(314)
@@ -231,7 +230,7 @@ func TestCanonicalMappingStorage(t *testing.T) {
 
 // Tests that head headers and head blocks can be assigned, individually.
 func TestHeadStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")})
 	blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")})
@@ -266,7 +265,7 @@ func TestHeadStorage(t *testing.T) {
 
 // Tests that receipts associated with a single block can be stored and retrieved.
 func TestBlockReceiptStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	receipt1 := &types.Receipt{
 		Status:            types.ReceiptStatusFailed,
@@ -279,6 +278,7 @@ func TestBlockReceiptStorage(t *testing.T) {
 		ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}),
 		GasUsed:         111111,
 	}
+	receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1})
 	receipt2 := &types.Receipt{
 		PostState:         common.Hash{2}.Bytes(),
 		CumulativeGasUsed: 2,
@@ -290,6 +290,7 @@ func TestBlockReceiptStorage(t *testing.T) {
 		ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}),
 		GasUsed:         222222,
 	}
+	receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2})
 	receipts := []*types.Receipt{receipt1, receipt2}
 
 	// Check that no receipt entries are in a pristine database

core/rawdb/accessors_indexes.go

@@ -19,88 +19,100 @@ package rawdb
 import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // ReadTxLookupEntry retrieves the positional metadata associated with a transaction
 // hash to allow retrieving the transaction or receipt by hash.
-func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) {
+func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) common.Hash {
 	data, _ := db.Get(txLookupKey(hash))
 	if len(data) == 0 {
-		return common.Hash{}, 0, 0
+		return common.Hash{}
 	}
-	var entry TxLookupEntry
+	if len(data) == common.HashLength {
+		return common.BytesToHash(data)
+	}
+	// Probably it's legacy txlookup entry data, try to decode it.
+	var entry LegacyTxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
-		log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err)
+		log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
-		return common.Hash{}, 0, 0
+		return common.Hash{}
 	}
-	return entry.BlockHash, entry.BlockIndex, entry.Index
+	return entry.BlockHash
 }
 
 // WriteTxLookupEntries stores a positional metadata for every transaction from
 // a block, enabling hash based transaction and receipt lookups.
-func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) {
+func WriteTxLookupEntries(db ethdb.Writer, block *types.Block) {
-	for i, tx := range block.Transactions() {
+	for _, tx := range block.Transactions() {
-		entry := TxLookupEntry{
+		if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil {
-			BlockHash:  block.Hash(),
-			BlockIndex: block.NumberU64(),
-			Index:      uint64(i),
-		}
-		data, err := rlp.EncodeToBytes(entry)
-		if err != nil {
-			log.Crit("Failed to encode transaction lookup entry", "err", err)
-		}
-		if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
 			log.Crit("Failed to store transaction lookup entry", "err", err)
 		}
 	}
 }
 
 // DeleteTxLookupEntry removes all transaction data associated with a hash.
-func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) {
+func DeleteTxLookupEntry(db ethdb.Deleter, hash common.Hash) {
 	db.Delete(txLookupKey(hash))
 }
 
 // ReadTransaction retrieves a specific transaction from the database, along with
 // its added positional metadata.
-func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
+func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
-	blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
+	blockHash := ReadTxLookupEntry(db, hash)
 	if blockHash == (common.Hash{}) {
 		return nil, common.Hash{}, 0, 0
 	}
-	body := ReadBody(db, blockHash, blockNumber)
+	blockNumber := ReadHeaderNumber(db, blockHash)
-	if body == nil || len(body.Transactions) <= int(txIndex) {
+	if blockNumber == nil {
-		log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex)
 		return nil, common.Hash{}, 0, 0
 	}
-	return body.Transactions[txIndex], blockHash, blockNumber, txIndex
+	body := ReadBody(db, blockHash, *blockNumber)
+	if body == nil {
+		log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
+		return nil, common.Hash{}, 0, 0
+	}
+	for txIndex, tx := range body.Transactions {
+		if tx.Hash() == hash {
+			return tx, blockHash, *blockNumber, uint64(txIndex)
+		}
+	}
+	log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
+	return nil, common.Hash{}, 0, 0
 }
 
 // ReadReceipt retrieves a specific transaction receipt from the database, along with
 // its added positional metadata.
-func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) {
+func ReadReceipt(db ethdb.Reader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) {
-	blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash)
+	blockHash := ReadTxLookupEntry(db, hash)
 	if blockHash == (common.Hash{}) {
 		return nil, common.Hash{}, 0, 0
 	}
-	receipts := ReadReceipts(db, blockHash, blockNumber)
+	blockNumber := ReadHeaderNumber(db, blockHash)
-	if len(receipts) <= int(receiptIndex) {
+	if blockNumber == nil {
-		log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex)
 		return nil, common.Hash{}, 0, 0
 	}
-	return receipts[receiptIndex], blockHash, blockNumber, receiptIndex
+	receipts := ReadReceipts(db, blockHash, *blockNumber)
+	for receiptIndex, receipt := range receipts {
+		if receipt.TxHash == hash {
+			return receipt, blockHash, *blockNumber, uint64(receiptIndex)
+		}
+	}
+	log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
+	return nil, common.Hash{}, 0, 0
 }
 
 // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given
 // section and bit index from the.
-func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) {
+func ReadBloomBits(db ethdb.Reader, bit uint, section uint64, head common.Hash) ([]byte, error) {
 	return db.Get(bloomBitsKey(bit, section, head))
 }
 
 // WriteBloomBits stores the compressed bloom bits vector belonging to the given
 // section and bit index.
-func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) {
+func WriteBloomBits(db ethdb.Writer, bit uint, section uint64, head common.Hash, bits []byte) {
 	if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil {
 		log.Crit("Failed to store bloom bits", "err", err)
 	}

core/rawdb/accessors_indexes_test.go

@@ -22,12 +22,12 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // Tests that positional lookup metadata can be stored and retrieved.
 func TestLookupStorage(t *testing.T) {
-	db := ethdb.NewMemDatabase()
+	db := NewMemoryDatabase()
 
 	tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
 	tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
@@ -65,4 +65,26 @@ func TestLookupStorage(t *testing.T) {
 			t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
 		}
 	}
+	// Insert legacy txlookup and verify the data retrieval
+	for index, tx := range block.Transactions() {
+		entry := LegacyTxLookupEntry{
+			BlockHash:  block.Hash(),
+			BlockIndex: block.NumberU64(),
+			Index:      uint64(index),
+		}
+		data, _ := rlp.EncodeToBytes(entry)
+		db.Put(txLookupKey(tx.Hash()), data)
+	}
+	for i, tx := range txs {
+		if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
+			t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
+		} else {
+			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
+				t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
+			}
+			if tx.Hash() != txn.Hash() {
+				t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
+			}
+		}
+	}
 }

core/rawdb/accessors_metadata.go

@@ -20,13 +20,14 @@ import (
 	"encoding/json"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // ReadDatabaseVersion retrieves the version number of the database.
-func ReadDatabaseVersion(db DatabaseReader) *uint64 {
+func ReadDatabaseVersion(db ethdb.Reader) *uint64 {
 	var version uint64
 
 	enc, _ := db.Get(databaseVerisionKey)
@@ -41,7 +42,7 @@ func ReadDatabaseVersion(db DatabaseReader) *uint64 {
 }
 
 // WriteDatabaseVersion stores the version number of the database
-func WriteDatabaseVersion(db DatabaseWriter, version uint64) {
+func WriteDatabaseVersion(db ethdb.Writer, version uint64) {
 	enc, err := rlp.EncodeToBytes(version)
 	if err != nil {
 		log.Crit("Failed to encode database version", "err", err)
@@ -52,7 +53,7 @@ func WriteDatabaseVersion(db DatabaseWriter, version uint64) {
 }
 
 // ReadChainConfig retrieves the consensus settings based on the given genesis hash.
-func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig {
+func ReadChainConfig(db ethdb.Reader, hash common.Hash) *params.ChainConfig {
 	data, _ := db.Get(configKey(hash))
 	if len(data) == 0 {
 		return nil
@@ -66,7 +67,7 @@ func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig {
 }
 
 // WriteChainConfig writes the chain config settings to the database.
-func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) {
+func WriteChainConfig(db ethdb.Writer, hash common.Hash, cfg *params.ChainConfig) {
 	if cfg == nil {
 		return
 	}
@@ -80,13 +81,13 @@ func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConf
 }
 
 // ReadPreimage retrieves a single preimage of the provided hash.
-func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
+func ReadPreimage(db ethdb.Reader, hash common.Hash) []byte {
 	data, _ := db.Get(preimageKey(hash))
 	return data
 }
 
 // WritePreimages writes the provided set of preimages to the database.
-func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
+func WritePreimages(db ethdb.Writer, preimages map[common.Hash][]byte) {
 	for hash, preimage := range preimages {
 		if err := db.Put(preimageKey(hash), preimage); err != nil {
 			log.Crit("Failed to store trie preimage", "err", err)

core/rawdb/database.go

@@ -0,0 +1,52 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/leveldb"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+)
+
+// NewDatabase creates a high level database on top of a given key-value data
+// store without a freezer moving immutable chain segments into cold storage.
+func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
+	return db
+}
+
+// NewMemoryDatabase creates an ephemeral in-memory key-value database without a
+// freezer moving immutable chain segments into cold storage.
+func NewMemoryDatabase() ethdb.Database {
+	return NewDatabase(memorydb.New())
+}
+
+// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database with
+// an initial starting capacity, but without a freezer moving immutable chain
+// segments into cold storage.
+func NewMemoryDatabaseWithCap(size int) ethdb.Database {
+	return NewDatabase(memorydb.NewWithCap(size))
+}
+
+// NewLevelDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) {
+	db, err := leveldb.New(file, cache, handles, namespace)
+	if err != nil {
+		return nil, err
+	}
+	return NewDatabase(db), nil
+}

core/rawdb/schema.go

@@ -63,9 +63,9 @@ var (
 	preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
 )
 
-// TxLookupEntry is a positional metadata to help looking up the data content of
+// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
-// a transaction or receipt given only its hash.
+// fields.
-type TxLookupEntry struct {
+type LegacyTxLookupEntry struct {
 	BlockHash  common.Hash
 	BlockIndex uint64
 	Index      uint64
@@ -78,6 +78,11 @@ func encodeBlockNumber(number uint64) []byte {
 	return enc
 }
 
+// headerKeyPrefix = headerPrefix + num (uint64 big endian)
+func headerKeyPrefix(number uint64) []byte {
+	return append(headerPrefix, encodeBlockNumber(number)...)
+}
+
 // headerKey = headerPrefix + num (uint64 big endian) + hash
 func headerKey(number uint64, hash common.Hash) []byte {
 	return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)

core/rawdb/table.go

@@ -0,0 +1,150 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// table is a wrapper around a database that prefixes each key access with a pre-
+// configured string.
+type table struct {
+	db     ethdb.Database
+	prefix string
+}
+
+// NewTable returns a database object that prefixes all keys with a given string.
+func NewTable(db ethdb.Database, prefix string) ethdb.Database {
+	return &table{
+		db:     db,
+		prefix: prefix,
+	}
+}
+
+// Close is a noop to implement the Database interface.
+func (t *table) Close() error {
+	return nil
+}
+
+// Has retrieves if a prefixed version of a key is present in the database.
+func (t *table) Has(key []byte) (bool, error) {
+	return t.db.Has(append([]byte(t.prefix), key...))
+}
+
+// Get retrieves the given prefixed key if it's present in the database.
+func (t *table) Get(key []byte) ([]byte, error) {
+	return t.db.Get(append([]byte(t.prefix), key...))
+}
+
+// Put inserts the given value into the database at a prefixed version of the
+// provided key.
+func (t *table) Put(key []byte, value []byte) error {
+	return t.db.Put(append([]byte(t.prefix), key...), value)
+}
+
+// Delete removes the given prefixed key from the database.
+func (t *table) Delete(key []byte) error {
+	return t.db.Delete(append([]byte(t.prefix), key...))
+}
+
+// NewIterator creates a binary-alphabetical iterator over the entire keyspace
+// contained within the database.
+func (t *table) NewIterator() ethdb.Iterator {
+	return t.NewIteratorWithPrefix(nil)
+}
+
+// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix.
+func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
+	return t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...))
+}
+
+// Stat returns a particular internal stat of the database.
+func (t *table) Stat(property string) (string, error) {
+	return t.db.Stat(property)
+}
+
+// Compact flattens the underlying data store for the given key range. In essence,
+// deleted and overwritten versions are discarded, and the data is rearranged to
+// reduce the cost of operations needed to access them.
+//
+// A nil start is treated as a key before all keys in the data store; a nil limit
+// is treated as a key after all keys in the data store. If both is nil then it
+// will compact entire data store.
+func (t *table) Compact(start []byte, limit []byte) error {
+	// If no start was specified, use the table prefix as the first value
+	if start == nil {
+		start = []byte(t.prefix)
+	}
+	// If no limit was specified, use the first element not matching the prefix
+	// as the limit
+	if limit == nil {
+		limit = []byte(t.prefix)
+		for i := len(limit) - 1; i >= 0; i-- {
+			// Bump the current character, stopping if it doesn't overflow
+			limit[i]++
+			if limit[i] > 0 {
+				break
+			}
+			// Character overflown, proceed to the next or nil if the last
+			if i == 0 {
+				limit = nil
+			}
+		}
+	}
+	// Range correctly calculated based on table prefix, delegate down
+	return t.db.Compact(start, limit)
+}
+
+// NewBatch creates a write-only database that buffers changes to its host db
+// until a final write is called, each operation prefixing all keys with the
+// pre-configured string.
+func (t *table) NewBatch() ethdb.Batch {
+	return &tableBatch{t.db.NewBatch(), t.prefix}
+}
+
+// tableBatch is a wrapper around a database batch that prefixes each key access
+// with a pre-configured string.
+type tableBatch struct {
+	batch  ethdb.Batch
+	prefix string
+}
+
+// Put inserts the given value into the batch for later committing.
+func (b *tableBatch) Put(key, value []byte) error {
+	return b.batch.Put(append([]byte(b.prefix), key...), value)
+}
+
+// Delete inserts the a key removal into the batch for later committing.
+func (b *tableBatch) Delete(key []byte) error {
+	return b.batch.Delete(append([]byte(b.prefix), key...))
+}
+
+// ValueSize retrieves the amount of data queued up for writing.
+func (b *tableBatch) ValueSize() int {
+	return b.batch.ValueSize()
+}
+
+// Write flushes any accumulated data to disk.
+func (b *tableBatch) Write() error {
+	return b.batch.Write()
+}
+
+// Reset resets the batch for reuse.
+func (b *tableBatch) Reset() {
+	b.batch.Reset()
+}

core/state/database.go

@@ -18,7 +18,6 @@ package state
 
 import (
 	"fmt"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -26,14 +25,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-// Trie cache generation limit after which to evict trie nodes from memory.
-var MaxTrieCacheGen = uint16(120)
-
 const (
-	// Number of past tries to keep. This value is chosen such that
-	// reasonable chain reorg depths will hit an existing trie.
-	maxPastTries = 12
-
 	// Number of codehash->size associations to keep.
 	codeSizeCacheSize = 100000
 )
@@ -59,28 +51,61 @@ type Database interface {
 	TrieDB() *trie.Database
 }
 
-// Trie is a Ethereum Merkle Trie.
+// Trie is a Ethereum Merkle Patricia trie.
 type Trie interface {
+	// GetKey returns the sha3 preimage of a hashed key that was previously used
+	// to store a value.
+	//
+	// TODO(fjl): remove this when SecureTrie is removed
+	GetKey([]byte) []byte
+
+	// TryGet returns the value for key stored in the trie. The value bytes must
+	// not be modified by the caller. If a node was not found in the database, a
+	// trie.MissingNodeError is returned.
 	TryGet(key []byte) ([]byte, error)
+
+	// TryUpdate associates key with value in the trie. If value has length zero, any
+	// existing value is deleted from the trie. The value bytes must not be modified
+	// by the caller while they are stored in the trie. If a node was not found in the
+	// database, a trie.MissingNodeError is returned.
 	TryUpdate(key, value []byte) error
+
+	// TryDelete removes any existing value for key from the trie. If a node was not
+	// found in the database, a trie.MissingNodeError is returned.
 	TryDelete(key []byte) error
-	Commit(onleaf trie.LeafCallback) (common.Hash, error)
+
+	// Hash returns the root hash of the trie. It does not write to the database and
+	// can be used even if the trie doesn't have one.
 	Hash() common.Hash
+
+	// Commit writes all nodes to the trie's memory database, tracking the internal
+	// and external (for account tries) references.
+	Commit(onleaf trie.LeafCallback) (common.Hash, error)
+
+	// NodeIterator returns an iterator that returns nodes of the trie. Iteration
+	// starts at the key after the given start key.
 	NodeIterator(startKey []byte) trie.NodeIterator
-	GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed
+
-	Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error
+	// Prove constructs a Merkle proof for key. The result contains all encoded nodes
+	// on the path to the value at key. The value itself is also included in the last
+	// node and can be retrieved by verifying the proof.
+	//
+	// If the trie does not contain a value for key, the returned proof contains all
+	// nodes of the longest existing prefix of the key (at least the root), ending
+	// with the node that proves the absence of the key.
+	Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains a few recent expanded trie nodes in memory. To keep
+// concurrent use, but does not retain any recent trie nodes in memory. To keep some
-// more historical state in memory, use the NewDatabaseWithCache constructor.
+// historical state in memory, use the NewDatabaseWithCache constructor.
 func NewDatabase(db ethdb.Database) Database {
 	return NewDatabaseWithCache(db, 0)
 }
 
-// NewDatabase creates a backing store for state. The returned database is safe for
+// NewDatabaseWithCache creates a backing store for state. The returned database
-// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a
-// well as a lot of collapsed RLP trie nodes in a large memory cache.
+// large memory cache.
 func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	return &cachingDB{
@@ -91,50 +116,22 @@ func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 
 type cachingDB struct {
 	db            *trie.Database
-	mu            sync.Mutex
-	pastTries     []*trie.SecureTrie
 	codeSizeCache *lru.Cache
 }
 
-// OpenTrie opens the main account trie.
+// OpenTrie opens the main account trie at a specific root hash.
 func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
-	db.mu.Lock()
+	return trie.NewSecure(root, db.db)
-	defer db.mu.Unlock()
-
-	for i := len(db.pastTries) - 1; i >= 0; i-- {
-		if db.pastTries[i].Hash() == root {
-			return cachedTrie{db.pastTries[i].Copy(), db}, nil
-		}
-	}
-	tr, err := trie.NewSecure(root, db.db, MaxTrieCacheGen)
-	if err != nil {
-		return nil, err
-	}
-	return cachedTrie{tr, db}, nil
-}
-
-func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
-	db.mu.Lock()
-	defer db.mu.Unlock()
-
-	if len(db.pastTries) >= maxPastTries {
-		copy(db.pastTries, db.pastTries[1:])
-		db.pastTries[len(db.pastTries)-1] = t
-	} else {
-		db.pastTries = append(db.pastTries, t)
-	}
 }
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
-	return trie.NewSecure(root, db.db, 0)
+	return trie.NewSecure(root, db.db)
 }
 
 // CopyTrie returns an independent copy of the given trie.
 func (db *cachingDB) CopyTrie(t Trie) Trie {
 	switch t := t.(type) {
-	case cachedTrie:
-		return cachedTrie{t.SecureTrie.Copy(), db}
 	case *trie.SecureTrie:
 		return t.Copy()
 	default:
@@ -164,21 +161,3 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 func (db *cachingDB) TrieDB() *trie.Database {
 	return db.db
 }
-
-// cachedTrie inserts its trie into a cachingDB on commit.
-type cachedTrie struct {
-	*trie.SecureTrie
-	db *cachingDB
-}
-
-func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
-	root, err := m.SecureTrie.Commit(onleaf)
-	if err == nil {
-		m.db.pushTrie(m.SecureTrie)
-	}
-	return root, err
-}
-
-func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
-	return m.SecureTrie.Prove(key, fromLevel, proofDb)
-}

core/state/iterator_test.go

@@ -51,7 +51,9 @@ func TestNodeIteratorCoverage(t *testing.T) {
 			t.Errorf("state entry not reported %x", hash)
 		}
 	}
-	for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() {
+	it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator()
+	for it.Next() {
+		key := it.Key()
 		if bytes.HasPrefix(key, []byte("secure-key-")) {
 			continue
 		}
@@ -59,4 +61,5 @@ func TestNodeIteratorCoverage(t *testing.T) {
 			t.Errorf("state entry not reported %x", key)
 		}
 	}
+	it.Release()
 }

core/state/managed_state_test.go

@@ -20,13 +20,13 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 )
 
 var addr = common.BytesToAddress([]byte("test"))
 
 func create() (*ManagedState, *account) {
-	statedb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
 	ms := ManageState(statedb)
 	ms.StateDB.SetNonce(addr, 100)
 	ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr))

core/state/state_test.go

@@ -22,13 +22,14 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	checker "gopkg.in/check.v1"
 )
 
 type StateSuite struct {
-	db    *ethdb.MemDatabase
+	db    ethdb.Database
 	state *StateDB
 }
 
@@ -87,7 +88,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 }
 
 func (s *StateSuite) SetUpTest(c *checker.C) {
-	s.db = ethdb.NewMemDatabase()
+	s.db = rawdb.NewMemoryDatabase()
 	s.state, _ = New(common.Hash{}, NewDatabase(s.db))
 }
 
@@ -141,7 +142,7 @@ func (s *StateSuite) TestSnapshotEmpty(c *checker.C) {
 // use testing instead of checker because checker does not support
 // printing/logging in tests (-check.vv does not work)
 func TestSnapshot2(t *testing.T) {
-	state, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
+	state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
 
 	stateobjaddr0 := toAddr([]byte("so0"))
 	stateobjaddr1 := toAddr([]byte("so1"))

core/state/statedb.go

@@ -37,8 +37,8 @@ type revision struct {
 }
 
 var (
-	// emptyState is the known hash of an empty state trie entry.
+	// emptyRoot is the known root hash of an empty trie.
-	emptyState = crypto.Keccak256Hash(nil)
+	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 
 	// emptyCode is the known hash of the empty EVM bytecode.
 	emptyCode = crypto.Keccak256Hash(nil)
@@ -653,7 +653,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
 		if err := rlp.DecodeBytes(leaf, &account); err != nil {
 			return nil
 		}
-		if account.Root != emptyState {
+		if account.Root != emptyRoot {
 			s.db.TrieDB().Reference(account.Root, parent)
 		}
 		code := common.BytesToHash(account.CodeHash)
@@ -662,6 +662,5 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
 		}
 		return nil
 	})
-	log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads())
 	return root, err
 }

core/state/statedb_test.go

@@ -31,15 +31,15 @@ import (
 	check "gopkg.in/check.v1"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // Tests that updating a state trie does not leak any database writes prior to
 // actually committing the state.
 func TestUpdateLeaks(t *testing.T) {
 	// Create an empty state database
-	db := ethdb.NewMemDatabase()
+	db := rawdb.NewMemoryDatabase()
 	state, _ := New(common.Hash{}, NewDatabase(db))
 
 	// Update it with some accounts
@@ -56,18 +56,19 @@ func TestUpdateLeaks(t *testing.T) {
 		state.IntermediateRoot(false)
 	}
 	// Ensure that no data was leaked into the database
-	for _, key := range db.Keys() {
+	it := db.NewIterator()
-		value, _ := db.Get(key)
+	for it.Next() {
-		t.Errorf("State leaked into database: %x -> %x", key, value)
+		t.Errorf("State leaked into database: %x -> %x", it.Key(), it.Value())
 	}
+	it.Release()
 }
 
 // Tests that no intermediate state of an object is stored into the database,
 // only the one right before the commit.
 func TestIntermediateLeaks(t *testing.T) {
 	// Create two state databases, one transitioning to the final state, the other final from the beginning
-	transDb := ethdb.NewMemDatabase()
+	transDb := rawdb.NewMemoryDatabase()
-	finalDb := ethdb.NewMemDatabase()
+	finalDb := rawdb.NewMemoryDatabase()
 	transState, _ := New(common.Hash{}, NewDatabase(transDb))
 	finalState, _ := New(common.Hash{}, NewDatabase(finalDb))
 
@@ -103,16 +104,20 @@ func TestIntermediateLeaks(t *testing.T) {
 	if _, err := finalState.Commit(false); err != nil {
 		t.Fatalf("failed to commit final state: %v", err)
 	}
-	for _, key := range finalDb.Keys() {
+	it := finalDb.NewIterator()
+	for it.Next() {
+		key := it.Key()
 		if _, err := transDb.Get(key); err != nil {
-			val, _ := finalDb.Get(key)
+			t.Errorf("entry missing from the transition database: %x -> %x", key, it.Value())
-			t.Errorf("entry missing from the transition database: %x -> %x", key, val)
 		}
 	}
-	for _, key := range transDb.Keys() {
+	it.Release()
+
+	it = transDb.NewIterator()
+	for it.Next() {
+		key := it.Key()
 		if _, err := finalDb.Get(key); err != nil {
-			val, _ := transDb.Get(key)
+			t.Errorf("extra entry in the transition database: %x -> %x", key, it.Value())
-			t.Errorf("extra entry in the transition database: %x -> %x", key, val)
 		}
 	}
 }
@@ -122,7 +127,7 @@ func TestIntermediateLeaks(t *testing.T) {
 // https://github.com/ethereum/go-ethereum/pull/15549.
 func TestCopy(t *testing.T) {
 	// Create a random state test to copy and modify "independently"
-	orig, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
+	orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
 
 	for i := byte(0); i < 255; i++ {
 		obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@@ -342,7 +347,7 @@ func (test *snapshotTest) String() string {
 func (test *snapshotTest) run() bool {
 	// Run all actions and create snapshots.
 	var (
-		state, _     = New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
+		state, _     = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
 		snapshotRevs = make([]int, len(test.snapshots))
 		sindex       = 0
 	)
@@ -433,7 +438,7 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy.
 // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512
 func TestCopyOfCopy(t *testing.T) {
-	sdb, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
+	sdb, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
 	addr := common.HexToAddress("aaaa")
 	sdb.SetBalance(addr, big.NewInt(42))
 

core/state/sync.go

@@ -20,12 +20,13 @@ import (
 	"bytes"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
 // NewStateSync create a new state trie download scheduler.
-func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.Sync {
+func NewStateSync(root common.Hash, database ethdb.Reader) *trie.Sync {
 	var syncer *trie.Sync
 	callback := func(leaf []byte, parent common.Hash) error {
 		var obj Account

core/state/sync_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/trie"
@@ -38,7 +39,7 @@ type testAccount struct {
 // makeTestState create a sample test state to test node-wise reconstruction.
 func makeTestState() (Database, common.Hash, []*testAccount) {
 	// Create an empty state
-	db := NewDatabase(ethdb.NewMemDatabase())
+	db := NewDatabase(rawdb.NewMemoryDatabase())
 	state, _ := New(common.Hash{}, db)
 
 	// Fill it with some arbitrary data
@@ -124,7 +125,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
 // Tests that an empty state is not scheduled for syncing.
 func TestEmptyStateSync(t *testing.T) {
 	empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
-	if req := NewStateSync(empty, ethdb.NewMemDatabase()).Missing(1); len(req) != 0 {
+	if req := NewStateSync(empty, rawdb.NewMemoryDatabase()).Missing(1); len(req) != 0 {
 		t.Errorf("content requested for empty state: %v", req)
 	}
 }
@@ -139,7 +140,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
 	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
-	dstDb := ethdb.NewMemDatabase()
+	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb)
 
 	queue := append([]common.Hash{}, sched.Missing(batch)...)
@@ -171,7 +172,7 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
-	dstDb := ethdb.NewMemDatabase()
+	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb)
 
 	queue := append([]common.Hash{}, sched.Missing(0)...)
@@ -208,7 +209,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
 	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
-	dstDb := ethdb.NewMemDatabase()
+	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb)
 
 	queue := make(map[common.Hash]struct{})
@@ -248,7 +249,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 	srcDb, srcRoot, srcAccounts := makeTestState()
 
 	// Create a destination state and sync with the scheduler
-	dstDb := ethdb.NewMemDatabase()
+	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb)
 
 	queue := make(map[common.Hash]struct{})
@@ -295,7 +296,7 @@ func TestIncompleteStateSync(t *testing.T) {
 	checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot)
 
 	// Create a destination state and sync with the scheduler
-	dstDb := ethdb.NewMemDatabase()
+	dstDb := rawdb.NewMemoryDatabase()
 	sched := NewStateSync(srcRoot, dstDb)
 
 	added := []common.Hash{}

core/tx_pool_test.go

@@ -27,10 +27,10 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -78,7 +78,7 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec
 }
 
 func setupTxPool() (*TxPool, *ecdsa.PrivateKey) {
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	key, _ := crypto.GenerateKey()
@@ -126,7 +126,7 @@ func validateEvents(events chan NewTxsEvent, count int) error {
 		case ev := <-events:
 			received = append(received, ev.Txs...)
 		case <-time.After(time.Second):
-			return fmt.Errorf("event #%d not fired", received)
+			return fmt.Errorf("event #%d not fired", len(received))
 		}
 	}
 	if len(received) > count {
@@ -163,7 +163,7 @@ func (c *testChain) State() (*state.StateDB, error) {
 	// a state change between those fetches.
 	stdb := c.statedb
 	if *c.trigger {
-		c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 		// simulate that the new head block included tx0 and tx1
 		c.statedb.SetNonce(c.address, 2)
 		c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether))
@@ -181,7 +181,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
 	var (
 		key, _     = crypto.GenerateKey()
 		address    = crypto.PubkeyToAddress(key.PublicKey)
-		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 		trigger    = false
 	)
 
@@ -335,7 +335,7 @@ func TestTransactionChainFork(t *testing.T) {
 
 	addr := crypto.PubkeyToAddress(key.PublicKey)
 	resetState := func() {
-		statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 		statedb.AddBalance(addr, big.NewInt(100000000000000))
 
 		pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@@ -364,7 +364,7 @@ func TestTransactionDoubleNonce(t *testing.T) {
 
 	addr := crypto.PubkeyToAddress(key.PublicKey)
 	resetState := func() {
-		statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 		statedb.AddBalance(addr, big.NewInt(100000000000000))
 
 		pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@@ -554,7 +554,7 @@ func TestTransactionPostponing(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the postponing with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -769,7 +769,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) {
 	t.Parallel()
 
 	// Create the pool to test the limit enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -857,7 +857,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) {
 	evictionInterval = time.Second
 
 	// Create the pool to test the non-expiration enforcement
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1011,7 +1011,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the limit enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1057,7 +1057,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the limit enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1091,7 +1091,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the limit enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1139,7 +1139,7 @@ func TestTransactionPoolRepricing(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the pricing enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1260,7 +1260,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the pricing enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1322,7 +1322,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the pricing enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1428,7 +1428,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the pricing enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1494,7 +1494,7 @@ func TestTransactionReplacement(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the pricing enforcement with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@@ -1588,7 +1588,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
 	os.Remove(journal)
 
 	// Create the original pool to inject transaction into the journal
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	config := testTxPoolConfig
@@ -1686,7 +1686,7 @@ func TestTransactionStatusCheck(t *testing.T) {
 	t.Parallel()
 
 	// Create the pool to test the status retrievals with
-	statedb, _ := state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
 	blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
 
 	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)

core/types.go

@@ -32,7 +32,7 @@ type Validator interface {
 
 	// ValidateState validates the given statedb and optionally the receipts and
 	// gas used.
-	ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error
+	ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error
 }
 
 // Processor is an interface for processing blocks using a given initial state.