diff --git a/.gitignore b/.gitignore index 0763d8492c..b8d2929016 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ profile.cov # IdeaIDE .idea +# VS Code +.vscode + # dashboard /dashboard/assets/flow-typed /dashboard/assets/node_modules diff --git a/.mailmap b/.mailmap index d51c7b6093..cc4b871a3c 100644 --- a/.mailmap +++ b/.mailmap @@ -65,7 +65,8 @@ Enrique Fynn Vincent G -RJ Catalano +RJ Catalano +RJ Catalano Nchinda Nchinda @@ -109,3 +110,14 @@ Frank Wang Gary Rong Guillaume Nicolas + +Sorin Neacsu +Sorin Neacsu + +Valentin Wüstholz +Valentin Wüstholz + +Armin Braun + +Ernesto del Toro +Ernesto del Toro diff --git a/.travis.yml b/.travis.yml index ba62b87bf5..a76a78954d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.7.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage - - os: linux dist: trusty sudo: required @@ -185,6 +174,8 @@ matrix: - xctool -version - xcrun simctl list + # Workaround for https://github.com/golang/go/issues/23749 + - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc' - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds # This builder does the Azure archive purges to avoid accumulating junk diff --git a/AUTHORS b/AUTHORS index faa19d281c..bd44a3de55 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,85 +1,173 @@ # This is the official list of go-ethereum authors for copyright purposes. +Afri Schoedon <5chdn@users.noreply.github.com> +Agustin Armellini Fischer +Airead +Alan Chen +Alejandro Isaza Ales Katona Alex Leverington +Alex Wu Alexandre Van de Sande +Ali Hajimirza +Anton Evangelatov +Arba Sasmoyo +Armani Ferrante +Armin Braun Aron Fischer Bas van Kervel Benjamin Brent +Benoit Verkindt +Bo +Bo Ye +Bob Glickstein Brian Schroeder Casey Detrio +Chase Wright Christoph Jentzsch Daniel A. Nagy +Daniel Sloof +Darrel Herbst +Dave Appleton Diego Siqueira +Dmitry Shulyak +Egon Elbre +Elias Naur Elliot Shepherd Enrique Fynn +Ernesto del Toro Ethan Buchman +Eugene Valeyev +Evangelos Pappas +Evgeny Danilenko <6655321@bk.ru> Fabian Vogelsteller +Fabio Barone Fabio Berger +FaceHo Felix Lange +Fiisio Frank Wang +Furkan KAMACI Gary Rong +George Ornbo Gregg Dourgarian +Guillaume Ballet Guillaume Nicolas Gustav Simonsson Hao Bryan Cheng Henning Diedrich Isidoro Ghezzi +Ivan Daniluk Jae Kwon Jamie Pitts +Janoš Guljaš Jason Carver +Jay Guo Jeff R. Allen Jeffrey Wilcke Jens Agerberg +Jia Chenhui +Jim McDonald +Joel Burget Jonathan Brown Joseph Chow Justin Clark-Casey Justin Drake Kenji Siu Kobi Gurkan +Konrad Feldmeier +Kurkó Mihály +Kyuntae Ethan Kim Lefteris Karapetsas Leif Jurvetson +Leo Shklovskii Lewis Marshall +Lio李欧 Louis Holbrook Luca Zeug +Magicking Maran Hidskes Marek Kotewicz +Mark Martin Holst Swende Matthew Di Ferrante Matthew Wampler-Doty +Maximilian Meister Micah Zoltu +Michael Ruminer +Miguel Mota +Miya Chen Nchinda Nchinda Nick Dodson Nick Johnson +Nicolas Guillaume +Noman +Oli Bye +Paul Litvak Paulo L F Casaretto +Paweł Bylica Peter Pratscher +Petr Mikusek Péter Szilágyi -RJ Catalano +RJ Catalano Ramesh Nair Ricardo Catalinas Jiménez +Ricardo Domingos +Richard Hart +Rob +Robert Zaremba +Russ Cox Rémy Roy +S. Matthew English Shintaro Kaneko +Sorin Neacsu Stein Dekker +Steve Waldman Steven Roose Taylor Gerring Thomas Bocek +Ti Zhou Tosh Camille -Valentin Wüstholz +Valentin Wüstholz Victor Farazdagi Victor Tran Viktor Trón Ville Sundell Vincent G Vitalik Buterin +Vitaly V Vivek Anand Vlad Gluhovsky Yohann Léon Yoichi Hirai +Yondon Fu +Zach Zahoor Mohamed +Zoe Nolan Zsolt Felföldi +am2rican5 +ayeowch +b00ris +bailantaotao +baizhenxuan +bloonfield +changhong +evgk +ferhat elmas holisticode +jtakalai ken10100147 ligi +mark.lin +necaremus +njupt-moon <1015041018@njupt.edu.cn> +nkbai +rhaps107 +slumber1122 +sunxiaojun2014 +terasum +tsarpaul xiekeyang +yoza ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com> Максим Чусовлянов diff --git a/README.md b/README.md index 61e36afec4..3d0d4d35d4 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ Official golang implementation of the Ethereum protocol. [![API Reference]( https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 )](https://godoc.org/github.com/ethereum/go-ethereum) +[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) +[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) Automated builds are available for stable releases and the unstable master branch. @@ -56,16 +58,14 @@ the user doesn't care about years-old historical data, so we can fast-sync quick state of the network. To do so: ``` -$ geth --fast --cache=512 console +$ geth console ``` This command will: - * Start geth in fast sync mode (`--fast`), causing it to download more data in exchange for avoiding - processing the entire history of the Ethereum network, which is very CPU intensive. - * Bump the memory allowance of the database to 512MB (`--cache=512`), which can help significantly in - sync times especially for HDD users. This flag is optional and you can set it as high or as low as - you'd like, though we'd recommend the 512MB - 2GB range. + * Start geth in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to + download more data in exchange for avoiding processing the entire history of the Ethereum network, + which is very CPU intensive. * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). @@ -80,12 +80,11 @@ entire system. In other words, instead of attaching to the main network, you wan network with your node, which is fully equivalent to the main network, but with play-Ether only. ``` -$ geth --testnet --fast --cache=512 console +$ geth --testnet console ``` -The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they -are equally useful on the testnet too. Please see above for their explanations if you've skipped to -here. +The `console` subcommand have the exact same meaning as above and they are equally useful on the +testnet too. Please see above for their explanations if you've skipped to here. Specifying the `--testnet` flag however will reconfigure your Geth instance a bit: @@ -102,6 +101,14 @@ over between the main network and test network, you should make sure to always u for play-money and real-money. Unless you manually move accounts, Geth will by default correctly separate the two networks and will not make any accounts available between them.* +### Full node on the Rinkeby test network + +The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum. + +``` +$ geth --rinkeby console +``` + ### Configuration As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via: @@ -125,10 +132,10 @@ One of the quickest ways to get Ethereum up and running on your machine is by us ``` docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ -p 8545:8545 -p 30303:30303 \ - ethereum/client-go --fast --cache=512 + ethereum/client-go ``` -This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. +This will start geth in fast-sync mode with a DB memory allowance of 1GB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside. diff --git a/VERSION b/VERSION index 27f9cd322b..53adb84c82 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.0 +1.8.2 diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index abcb403db2..254b1f7fb4 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -136,11 +136,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { // MethodById looks up a method by the 4-byte id // returns nil if none found -func (abi *ABI) MethodById(sigdata []byte) *Method { +func (abi *ABI) MethodById(sigdata []byte) (*Method, error) { for _, method := range abi.Methods { if bytes.Equal(method.Id(), sigdata[:4]) { - return &method + return &method, nil } } - return nil + return nil, fmt.Errorf("no method with id: %#x", sigdata[:4]) } diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 2d43b631c2..35e0094ddd 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -689,7 +689,11 @@ func TestABI_MethodById(t *testing.T) { } for name, m := range abi.Methods { a := fmt.Sprintf("%v", m) - b := fmt.Sprintf("%v", abi.MethodById(m.Id())) + m2, err := abi.MethodById(m.Id()) + if err != nil { + t.Fatalf("Failed to look up ABI method: %v", err) + } + b := fmt.Sprintf("%v", m2) if a != b { t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id())) } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 04ca6150ad..f171f4cc63 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -67,6 +67,17 @@ func (arguments Arguments) LengthNonIndexed() int { return out } +// NonIndexed returns the arguments with indexed arguments filtered out +func (arguments Arguments) NonIndexed() Arguments { + var ret []Argument + for _, arg := range arguments { + if !arg.Indexed { + ret = append(ret, arg) + } + } + return ret +} + // isTuple returns true for non-atomic constructs, like (uint,uint) or uint[] func (arguments Arguments) isTuple() bool { return len(arguments) > 1 @@ -74,21 +85,25 @@ func (arguments Arguments) isTuple() bool { // Unpack performs the operation hexdata -> Go format func (arguments Arguments) Unpack(v interface{}, data []byte) error { - if arguments.isTuple() { - return arguments.unpackTuple(v, data) - } - return arguments.unpackAtomic(v, data) -} -func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { // make sure the passed value is arguments pointer - valueOf := reflect.ValueOf(v) - if reflect.Ptr != valueOf.Kind() { + if reflect.Ptr != reflect.ValueOf(v).Kind() { return fmt.Errorf("abi: Unpack(non-pointer %T)", v) } + marshalledValues, err := arguments.UnpackValues(data) + if err != nil { + return err + } + if arguments.isTuple() { + return arguments.unpackTuple(v, marshalledValues) + } + return arguments.unpackAtomic(v, marshalledValues) +} + +func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error { var ( - value = valueOf.Elem() + value = reflect.ValueOf(v).Elem() typ = value.Type() kind = value.Kind() ) @@ -110,30 +125,9 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { exists[field] = true } } - // `i` counts the nonindexed arguments. - // `j` counts the number of complex types. - // both `i` and `j` are used to to correctly compute `data` offset. + for i, arg := range arguments.NonIndexed() { - i, j := -1, 0 - for _, arg := range arguments { - - if arg.Indexed { - // can't read, continue - continue - } - i++ - marshalledValue, err := toGoType((i+j)*32, arg.Type, output) - if err != nil { - return err - } - - if arg.Type.T == ArrayTy { - // combined index ('i' + 'j') need to be adjusted only by size of array, thus - // we need to decrement 'j' because 'i' was incremented - j += arg.Type.Size - 1 - } - - reflectValue := reflect.ValueOf(marshalledValue) + reflectValue := reflect.ValueOf(marshalledValues[i]) switch kind { case reflect.Struct: @@ -166,34 +160,52 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { } // unpackAtomic unpacks ( hexdata -> go ) a single value -func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { - // make sure the passed value is arguments pointer - valueOf := reflect.ValueOf(v) - if reflect.Ptr != valueOf.Kind() { - return fmt.Errorf("abi: Unpack(non-pointer %T)", v) +func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error { + if len(marshalledValues) != 1 { + return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) } - arg := arguments[0] - if arg.Indexed { - return fmt.Errorf("abi: attempting to unpack indexed variable into element.") - } - - value := valueOf.Elem() - - marshalledValue, err := toGoType(0, arg.Type, output) - if err != nil { - return err - } - return set(value, reflect.ValueOf(marshalledValue), arg) + elem := reflect.ValueOf(v).Elem() + reflectValue := reflect.ValueOf(marshalledValues[0]) + return set(elem, reflectValue, arguments.NonIndexed()[0]) } -// Unpack performs the operation Go format -> Hexdata +// UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, +// without supplying a struct to unpack into. Instead, this method returns a list containing the +// values. An atomic argument will be a list with one element. +func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { + retval := make([]interface{}, 0, arguments.LengthNonIndexed()) + virtualArgs := 0 + for index, arg := range arguments.NonIndexed() { + marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data) + if arg.Type.T == ArrayTy { + // If we have a static array, like [3]uint256, these are coded as + // just like uint256,uint256,uint256. + // This means that we need to add two 'virtual' arguments when + // we count the index from now on + + virtualArgs += arg.Type.Size - 1 + } + if err != nil { + return nil, err + } + retval = append(retval, marshalledValue) + } + return retval, nil +} + +// PackValues performs the operation Go format -> Hexdata +// It is the semantic opposite of UnpackValues +func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) { + return arguments.Pack(args...) +} + +// Pack performs the operation Go format -> Hexdata func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // Make sure arguments match up and pack them abiArgs := arguments if len(args) != len(abiArgs) { return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(abiArgs)) } - // variable input is the output appended at the end of packed // output. This is used for strings and bytes types input. var variableInput []byte @@ -207,7 +219,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { inputOffset += 32 } } - var ret []byte for i, a := range args { input := abiArgs[i] @@ -216,7 +227,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if err != nil { return nil, err } - // check for a slice type (string, bytes, slice) if input.Type.requiresLengthPrefix() { // calculate the offset diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index bd342a8cb9..fe7dea4da1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -428,10 +428,23 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb } return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + if receipts == nil { + return nil, nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil +} + func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 36401ee677..14ab516ac2 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 80efb3f7ea..761c80edfd 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -95,6 +95,9 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) { // iteratively unpack elements func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) { + if size < 0 { + return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size) + } if start+32*size > len(output) { return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size) } @@ -181,16 +184,32 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // interprets a 32 byte slice as an offset and then determines which indice to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - offset := int(binary.BigEndian.Uint64(output[index+24 : index+32])) - if offset+32 > len(output) { - return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32) - } - length = int(binary.BigEndian.Uint64(output[offset+24 : offset+32])) - if offset+32+length > len(output) { - return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length) - } - start = offset + 32 + bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32]) + bigOffsetEnd.Add(bigOffsetEnd, common.Big32) + outputLength := big.NewInt(int64(len(output))) - //fmt.Printf("LENGTH PREFIX INFO: \nsize: %v\noffset: %v\nstart: %v\n", length, offset, start) + if bigOffsetEnd.Cmp(outputLength) > 0 { + return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", bigOffsetEnd, outputLength) + } + + if bigOffsetEnd.BitLen() > 63 { + return 0, 0, fmt.Errorf("abi offset larger than int64: %v", bigOffsetEnd) + } + + offsetEnd := int(bigOffsetEnd.Uint64()) + lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd]) + + totalSize := big.NewInt(0) + totalSize.Add(totalSize, bigOffsetEnd) + totalSize.Add(totalSize, lengthBig) + if totalSize.BitLen() > 63 { + return 0, 0, fmt.Errorf("abi length larger than int64: %v", totalSize) + } + + if totalSize.Cmp(outputLength) > 0 { + return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %v require %v", outputLength, totalSize) + } + start = int(bigOffsetEnd.Uint64()) + length = int(lengthBig.Uint64()) return } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 4d7fe638c4..742211244b 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -130,7 +130,7 @@ var unpackTests = []unpackTest{ { def: `[{"type": "bytes32"}]`, enc: "0100000000000000000000000000000000000000000000000000000000000000", - want: common.HexToHash("0100000000000000000000000000000000000000000000000000000000000000"), + want: [32]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, }, { def: `[{"type": "function"}]`, @@ -683,3 +683,73 @@ func TestUnmarshal(t *testing.T) { t.Fatal("expected error:", err) } } + +func TestOOMMaliciousInput(t *testing.T) { + oomTests := []unpackTest{ + { + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000003" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Length larger than 64 bits + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset very large (over 64 bits) + def: `[{"type": "uint8[]"}]`, + enc: "00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset very large (below 64 bits) + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000007ffffffffff00020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset negative (as 64 bit) + def: `[{"type": "uint8[]"}]`, + enc: "000000000000000000000000000000000000000000000000f000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + + { // Negative length + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "000000000000000000000000000000000000000000000000f000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Very large length + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "0000000000000000000000000000000000000000000000007fffffffff000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + } + for i, test := range oomTests { + def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(def)) + if err != nil { + t.Fatalf("invalid ABI definition %s: %v", def, err) + } + encb, err := hex.DecodeString(test.enc) + if err != nil { + t.Fatalf("invalid hex: %s" + test.enc) + } + _, err = abi.Methods["method"].Outputs.UnpackValues(encb) + if err == nil { + t.Fatalf("Expected error on malicious input, test %d", i) + } + } +} diff --git a/build/ci.go b/build/ci.go index 1f98bb8438..24b58c1ae4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -121,7 +121,8 @@ var ( // Note: vivid is unsupported because there is no golang-1.6 package for it. // Note: wily is unsupported because it was officially deprecated on lanchpad. // Note: yakkety is unsupported because it was officially deprecated on lanchpad. - debDistros = []string{"trusty", "xenial", "zesty", "artful"} + // Note: zesty is unsupported because it was officially deprecated on lanchpad. + debDistros = []string{"trusty", "xenial", "artful", "bionic"} ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -181,13 +182,13 @@ func doInstall(cmdline []string) { // Check Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. if !strings.Contains(runtime.Version(), "devel") { - // Figure out the minor version number since we can't textually compare (1.10 < 1.7) + // Figure out the minor version number since we can't textually compare (1.10 < 1.8) var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 7 { + if minor < 8 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.7 and cannot") + log.Println("go-ethereum requires at least Go version 1.8 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } diff --git a/build/update-license.go b/build/update-license.go index 3d69598b75..22e4033428 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -55,10 +55,9 @@ var ( "crypto/sha3/", "internal/jsre/deps", "log/", + "common/bitutil/bitutil", // don't license generated files - "contracts/chequebook/contract/", - "contracts/ens/contract/", - "contracts/release/contract.go", + "contracts/chequebook/contract/code.go", } // paths with this prefix are licensed as GPL. all other files are LGPL. diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index ecfc6fc24e..2e93cc04d2 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -122,7 +122,12 @@ func main() { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil { + cfg := discover.Config{ + PrivateKey: nodeKey, + AnnounceAddr: realaddr, + NetRestrict: restrictList, + } + if _, err := discover.ListenUDP(conn, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index 219a5460b8..dbf5afc0ce 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -1,3 +1,19 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( diff --git a/cmd/ethkey/message_test.go b/cmd/ethkey/message_test.go index fb16f03d02..39352b1d22 100644 --- a/cmd/ethkey/message_test.go +++ b/cmd/ethkey/message_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of go-ethereum. // // go-ethereum is free software: you can redistribute it and/or modify diff --git a/cmd/ethkey/run_test.go b/cmd/ethkey/run_test.go index 8ce4fe5cde..6006f6b5bb 100644 --- a/cmd/ethkey/run_test.go +++ b/cmd/ethkey/run_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of go-ethereum. // // go-ethereum is free software: you can redistribute it and/or modify diff --git a/cmd/evm/json_logger.go b/cmd/evm/json_logger.go index 47daf7dbbc..0e7a911896 100644 --- a/cmd/evm/json_logger.go +++ b/cmd/evm/json_logger.go @@ -1,18 +1,18 @@ // Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. +// This file is part of go-ethereum. // -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// go-ethereum is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. +// GNU General Public License for more details. // -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . package main diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 99527f9d1e..095668c868 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -686,8 +686,6 @@ func authTwitter(url string) (string, string, common.Address, error) { if len(parts) < 4 || parts[len(parts)-2] != "status" { return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } - username := parts[len(parts)-3] - // Twitter's API isn't really friendly with direct links. Still, we don't // want to do ask read permissions from users, so just load the public posts and // scrape it for the Ethereum address and profile URL. @@ -697,6 +695,13 @@ func authTwitter(url string) (string, string, common.Address, error) { } defer res.Body.Close() + // Resolve the username from the final redirect, no intermediate junk + parts = strings.Split(res.Request.URL.String(), "/") + if len(parts) < 4 || parts[len(parts)-2] != "status" { + return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + } + username := parts[len(parts)-3] + body, err := ioutil.ReadAll(res.Body) if err != nil { return "", "", common.Address{}, err diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 35bf576e1d..85d0c3acaa 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -67,6 +67,9 @@ It expects the genesis file as argument.`, utils.DataDirFlag, utils.CacheFlag, utils.LightModeFlag, + utils.GCModeFlag, + utils.CacheDatabaseFlag, + utils.CacheGCFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 9d5cc38a1b..2500a969cf 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -22,6 +22,7 @@ import ( "os/signal" "path/filepath" "strings" + "syscall" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" @@ -42,7 +43,7 @@ var ( Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`, +See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`, } attachCommand = cli.Command{ @@ -55,7 +56,7 @@ See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`, Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console. +See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console. This command allows to open a console on a running geth node.`, } @@ -68,7 +69,7 @@ This command allows to open a console on a running geth node.`, Category: "CONSOLE COMMANDS", Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp -JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console`, +JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console`, } ) @@ -207,7 +208,7 @@ func ephemeralConsole(ctx *cli.Context) error { } // Wait for pending callbacks, but stop for Ctrl-C. abort := make(chan os.Signal, 1) - signal.Notify(abort, os.Interrupt) + signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM) go func() { <-abort diff --git a/cmd/geth/main.go b/cmd/geth/main.go index cb8d63bf71..a82e5c89cf 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -114,6 +114,7 @@ var ( utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.RPCCORSDomainFlag, + utils.RPCVirtualHostsFlag, utils.EthStatsURLFlag, utils.MetricsEnabledFlag, utils.FakePoWFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index a2bcaff027..a1558c2330 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -156,6 +156,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.IPCDisabledFlag, utils.IPCPathFlag, utils.RPCCORSDomainFlag, + utils.RPCVirtualHostsFlag, utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag, diff --git a/cmd/p2psim/main.go b/cmd/p2psim/main.go index 56b74d135b..0c8ed038d5 100644 --- a/cmd/p2psim/main.go +++ b/cmd/p2psim/main.go @@ -1,3 +1,19 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + // p2psim provides a command-line client for a simulation HTTP API. // // Here is an example of creating a 2 node network with the first node diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index f747f47395..1974a94aa2 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -168,19 +168,18 @@ type parityChainSpec struct { Engine struct { Ethash struct { Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - HomesteadTransition uint64 `json:"homesteadTransition"` - EIP150Transition uint64 `json:"eip150Transition"` - EIP160Transition uint64 `json:"eip160Transition"` - EIP161abcTransition uint64 `json:"eip161abcTransition"` - EIP161dTransition uint64 `json:"eip161dTransition"` - EIP649Reward *hexutil.Big `json:"eip649Reward"` - EIP100bTransition uint64 `json:"eip100bTransition"` - EIP649Transition uint64 `json:"eip649Transition"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` + DurationLimit *hexutil.Big `json:"durationLimit"` + BlockReward *hexutil.Big `json:"blockReward"` + HomesteadTransition uint64 `json:"homesteadTransition"` + EIP150Transition uint64 `json:"eip150Transition"` + EIP160Transition uint64 `json:"eip160Transition"` + EIP161abcTransition uint64 `json:"eip161abcTransition"` + EIP161dTransition uint64 `json:"eip161dTransition"` + EIP649Reward *hexutil.Big `json:"eip649Reward"` + EIP100bTransition uint64 `json:"eip100bTransition"` + EIP649Transition uint64 `json:"eip649Transition"` } `json:"params"` } `json:"Ethash"` } `json:"engine"` @@ -188,6 +187,7 @@ type parityChainSpec struct { Params struct { MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` NetworkID hexutil.Uint64 `json:"networkID"` MaxCodeSize uint64 `json:"maxCodeSize"` EIP155Transition uint64 `json:"eip155Transition"` @@ -270,7 +270,6 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin } spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Engine.Ethash.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64() @@ -284,6 +283,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) + spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 1092c4c88e..3832b247f8 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -117,7 +117,7 @@ var dashboardContent = `

To run an archive node, download {{.GethGenesis}} and start Geth with:

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
-
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}
+
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -136,7 +136,7 @@ var dashboardContent = `

To run a full node, download {{.GethGenesis}} and start Geth with:

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
-
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}
+
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -158,7 +158,7 @@ var dashboardContent = `

To run a light node, download {{.GethGenesis}} and start Geth with:

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
-
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}
+
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -177,7 +177,7 @@ var dashboardContent = `

To run an embedded node, download {{.GethGenesis}} and start Geth with:

geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}
-
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}
+
geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}


You can download Geth from https://geth.ethereum.org/downloads/.

@@ -208,7 +208,7 @@ var dashboardContent = `
geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}

With your local chain initialized, you can start the Ethereum Wallet: -

ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}
+
ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}


You can download the Ethereum Wallet from https://github.com/ethereum/mist/releases.

@@ -229,7 +229,7 @@ var dashboardContent = `
geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}

With your local chain initialized, you can start Mist: -

mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}
+
mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}


You can download the Mist browser from https://github.com/ethereum/mist/releases.

@@ -261,7 +261,7 @@ var dashboardContent = `

Inside your Java code you can now import the geth archive and connect to Ethereum:

import org.ethereum.geth.*;
-Enodes bootnodes = new Enodes();{{range .BootnodesLight}}
+Enodes bootnodes = new Enodes();{{range .Bootnodes}}
 bootnodes.append(new Enode("{{.}}"));{{end}}
 
 NodeConfig config = new NodeConfig();
@@ -294,7 +294,7 @@ node.start();
 
 var error: NSError?
 
-let bootnodes = GethNewEnodesEmpty(){{range .BootnodesLight}}
+let bootnodes = GethNewEnodesEmpty(){{range .Bootnodes}}
 bootnodes?.append(GethNewEnode("{{.}}", &error)){{end}}
 
 let config = GethNewNodeConfig()
@@ -595,44 +595,43 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 		statsLogin = ""
 	}
 	indexfile := new(bytes.Buffer)
-	bootCpp := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootCpp := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootCpp[i] = "required:" + strings.TrimPrefix(boot, "enode://")
 	}
-	bootHarmony := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootHarmony := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootHarmony[i] = fmt.Sprintf("-Dpeer.active.%d.url=%s", i, boot)
 	}
-	bootPython := make([]string, len(conf.bootFull))
-	for i, boot := range conf.bootFull {
+	bootPython := make([]string, len(conf.bootnodes))
+	for i, boot := range conf.bootnodes {
 		bootPython[i] = "'" + boot + "'"
 	}
 	template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{
-		"Network":            network,
-		"NetworkID":          conf.Genesis.Config.ChainId,
-		"NetworkTitle":       strings.Title(network),
-		"EthstatsPage":       config.ethstats,
-		"ExplorerPage":       config.explorer,
-		"WalletPage":         config.wallet,
-		"FaucetPage":         config.faucet,
-		"GethGenesis":        network + ".json",
-		"BootnodesFull":      conf.bootFull,
-		"BootnodesLight":     conf.bootLight,
-		"BootnodesFullFlat":  strings.Join(conf.bootFull, ","),
-		"BootnodesLightFlat": strings.Join(conf.bootLight, ","),
-		"Ethstats":           statsLogin,
-		"Ethash":             conf.Genesis.Config.Ethash != nil,
-		"CppGenesis":         network + "-cpp.json",
-		"CppBootnodes":       strings.Join(bootCpp, " "),
-		"HarmonyGenesis":     network + "-harmony.json",
-		"HarmonyBootnodes":   strings.Join(bootHarmony, " "),
-		"ParityGenesis":      network + "-parity.json",
-		"PythonGenesis":      network + "-python.json",
-		"PythonBootnodes":    strings.Join(bootPython, ","),
-		"Homestead":          conf.Genesis.Config.HomesteadBlock,
-		"Tangerine":          conf.Genesis.Config.EIP150Block,
-		"Spurious":           conf.Genesis.Config.EIP155Block,
-		"Byzantium":          conf.Genesis.Config.ByzantiumBlock,
+		"Network":          network,
+		"NetworkID":        conf.Genesis.Config.ChainId,
+		"NetworkTitle":     strings.Title(network),
+		"EthstatsPage":     config.ethstats,
+		"ExplorerPage":     config.explorer,
+		"WalletPage":       config.wallet,
+		"FaucetPage":       config.faucet,
+		"GethGenesis":      network + ".json",
+		"Bootnodes":        conf.bootnodes,
+		"BootnodesFlat":    strings.Join(conf.bootnodes, ","),
+		"Ethstats":         statsLogin,
+		"Ethash":           conf.Genesis.Config.Ethash != nil,
+		"CppGenesis":       network + "-cpp.json",
+		"CppBootnodes":     strings.Join(bootCpp, " "),
+		"HarmonyGenesis":   network + "-harmony.json",
+		"HarmonyBootnodes": strings.Join(bootHarmony, " "),
+		"ParityGenesis":    network + "-parity.json",
+		"PythonGenesis":    network + "-python.json",
+		"PythonBootnodes":  strings.Join(bootPython, ","),
+		"Homestead":        conf.Genesis.Config.HomesteadBlock,
+		"Tangerine":        conf.Genesis.Config.EIP150Block,
+		"Spurious":         conf.Genesis.Config.EIP155Block,
+		"Byzantium":        conf.Genesis.Config.ByzantiumBlock,
+		"Constantinople":   conf.Genesis.Config.ConstantinopleBlock,
 	})
 	files[filepath.Join(workdir, "index.html")] = indexfile.Bytes()
 
@@ -651,7 +650,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 		harmonySpecJSON, _ := conf.Genesis.MarshalJSON()
 		files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON
 
-		paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootFull)
+		paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes)
 		if err != nil {
 			return nil, err
 		}
diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go
index 92b4cb2861..976bf04d00 100644
--- a/cmd/puppeth/module_faucet.go
+++ b/cmd/puppeth/module_faucet.go
@@ -93,7 +93,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"NetworkID":     config.node.network,
 		"Bootnodes":     strings.Join(bootnodes, ","),
 		"Ethstats":      config.node.ethstats,
-		"EthPort":       config.node.portFull,
+		"EthPort":       config.node.port,
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
 		"FaucetName":    strings.Title(network),
@@ -110,7 +110,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"Datadir":       config.node.datadir,
 		"VHost":         config.host,
 		"ApiPort":       config.port,
-		"EthPort":       config.node.portFull,
+		"EthPort":       config.node.port,
 		"EthName":       config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
 		"CaptchaToken":  config.captchaToken,
 		"CaptchaSecret": config.captchaSecret,
@@ -158,7 +158,7 @@ func (info *faucetInfos) Report() map[string]string {
 	report := map[string]string{
 		"Website address":              info.host,
 		"Website listener port":        strconv.Itoa(info.port),
-		"Ethereum listener port":       strconv.Itoa(info.node.portFull),
+		"Ethereum listener port":       strconv.Itoa(info.node.port),
 		"Funding amount (base tier)":   fmt.Sprintf("%d Ethers", info.amount),
 		"Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes),
 		"Funding tiers":                strconv.Itoa(info.tiers),
@@ -228,7 +228,7 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
 	return &faucetInfos{
 		node: &nodeInfos{
 			datadir:  infos.volumes["/root/.faucet"],
-			portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
+			port:     infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
 			ethstats: infos.envvars["ETH_NAME"],
 			keyJSON:  keyJSON,
 			keyPass:  keyPass,
diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go
index 69cb19c349..2609fd976e 100644
--- a/cmd/puppeth/module_node.go
+++ b/cmd/puppeth/module_node.go
@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -56,15 +56,13 @@ services:
     build: .
     image: {{.Network}}/{{.Type}}
     ports:
-      - "{{.FullPort}}:{{.FullPort}}"
-      - "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
-      - "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
+      - "{{.Port}}:{{.Port}}"
+      - "{{.Port}}:{{.Port}}/udp"
     volumes:
       - {{.Datadir}}:/root/.ethereum{{if .Ethashdir}}
       - {{.Ethashdir}}:/root/.ethash{{end}}
     environment:
-      - FULL_PORT={{.FullPort}}/tcp
-      - LIGHT_PORT={{.LightPort}}/udp
+      - PORT={{.Port}}/tcp
       - TOTAL_PEERS={{.TotalPeers}}
       - LIGHT_PEERS={{.LightPeers}}
       - STATS_NAME={{.Ethstats}}
@@ -82,12 +80,11 @@ services:
 // deployNode deploys a new Ethereum node container to a remote machine via SSH,
 // docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos, nocache bool) ([]byte, error) {
+func deployNode(client *sshClient, network string, bootnodes []string, config *nodeInfos, nocache bool) ([]byte, error) {
 	kind := "sealnode"
 	if config.keyJSON == "" && config.etherbase == "" {
 		kind = "bootnode"
-		bootv4 = make([]string, 0)
-		bootv5 = make([]string, 0)
+		bootnodes = make([]string, 0)
 	}
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
@@ -100,11 +97,10 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 	dockerfile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
-		"Port":      config.portFull,
+		"Port":      config.port,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
-		"BootV4":    strings.Join(bootv4, ","),
-		"BootV5":    strings.Join(bootv5, ","),
+		"Bootnodes": strings.Join(bootnodes, ","),
 		"Ethstats":  config.ethstats,
 		"Etherbase": config.etherbase,
 		"GasTarget": uint64(1000000 * config.gasTarget),
@@ -119,10 +115,9 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
 		"Datadir":    config.datadir,
 		"Ethashdir":  config.ethashdir,
 		"Network":    network,
-		"FullPort":   config.portFull,
+		"Port":       config.port,
 		"TotalPeers": config.peersTotal,
 		"Light":      config.peersLight > 0,
-		"LightPort":  config.portFull + 1,
 		"LightPeers": config.peersLight,
 		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
 		"Etherbase":  config.etherbase,
@@ -157,10 +152,8 @@ type nodeInfos struct {
 	datadir    string
 	ethashdir  string
 	ethstats   string
-	portFull   int
-	portLight  int
-	enodeFull  string
-	enodeLight string
+	port       int
+	enode      string
 	peersTotal int
 	peersLight int
 	etherbase  string
@@ -174,15 +167,11 @@ type nodeInfos struct {
 // most - but not all - fields for reporting to the user.
 func (info *nodeInfos) Report() map[string]string {
 	report := map[string]string{
-		"Data directory":             info.datadir,
-		"Listener port (full nodes)": strconv.Itoa(info.portFull),
-		"Peer count (all total)":     strconv.Itoa(info.peersTotal),
-		"Peer count (light nodes)":   strconv.Itoa(info.peersLight),
-		"Ethstats username":          info.ethstats,
-	}
-	if info.peersLight > 0 {
-		// Light server enabled
-		report["Listener port (light nodes)"] = strconv.Itoa(info.portLight)
+		"Data directory":           info.datadir,
+		"Listener port":            strconv.Itoa(info.port),
+		"Peer count (all total)":   strconv.Itoa(info.peersTotal),
+		"Peer count (light nodes)": strconv.Itoa(info.peersLight),
+		"Ethstats username":        info.ethstats,
 	}
 	if info.gasTarget > 0 {
 		// Miner or signer node
@@ -250,7 +239,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		keyPass = string(bytes.TrimSpace(out))
 	}
 	// Run a sanity check to see if the devp2p is reachable
-	port := infos.portmap[infos.envvars["FULL_PORT"]]
+	port := infos.portmap[infos.envvars["PORT"]]
 	if err = checkPort(client.server, port); err != nil {
 		log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
 	}
@@ -259,8 +248,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		genesis:    genesis,
 		datadir:    infos.volumes["/root/.ethereum"],
 		ethashdir:  infos.volumes["/root/.ethash"],
-		portFull:   infos.portmap[infos.envvars["FULL_PORT"]],
-		portLight:  infos.portmap[infos.envvars["LIGHT_PORT"]],
+		port:       port,
 		peersTotal: totalPeers,
 		peersLight: lightPeers,
 		ethstats:   infos.envvars["STATS_NAME"],
@@ -270,9 +258,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		gasTarget:  gasTarget,
 		gasPrice:   gasPrice,
 	}
-	stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
-	if stats.portLight != 0 {
-		stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
-	}
+	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+
 	return stats, nil
 }
diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go
index 2e2b4644cf..b88a61de7d 100644
--- a/cmd/puppeth/wizard.go
+++ b/cmd/puppeth/wizard.go
@@ -40,8 +40,7 @@ import (
 // between sessions.
 type config struct {
 	path      string   // File containing the configuration values
-	bootFull  []string // Bootnodes to always connect to by full nodes
-	bootLight []string // Bootnodes to always connect to by light nodes
+	bootnodes []string // Bootnodes to always connect to by all nodes
 	ethstats  string   // Ethstats settings to cache for node deploys
 
 	Genesis *core.Genesis     `json:"genesis,omitempty"` // Genesis block to cache for node deploys
diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go
index 10ef72f788..413511c1c3 100644
--- a/cmd/puppeth/wizard_explorer.go
+++ b/cmd/puppeth/wizard_explorer.go
@@ -55,7 +55,7 @@ func (w *wizard) deployExplorer() {
 	}
 	existed := err == nil
 
-	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootFull)
+	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes)
 	if err != nil {
 		log.Error("Failed to create chain spec for explorer", "err", err)
 		return
diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go
index 191575b168..9a429bc96d 100644
--- a/cmd/puppeth/wizard_faucet.go
+++ b/cmd/puppeth/wizard_faucet.go
@@ -38,7 +38,7 @@ func (w *wizard) deployFaucet() {
 	infos, err := checkFaucet(client, w.network)
 	if err != nil {
 		infos = &faucetInfos{
-			node:    &nodeInfos{portFull: 30303, peersTotal: 25},
+			node:    &nodeInfos{port: 30303, peersTotal: 25},
 			port:    80,
 			host:    client.server,
 			amount:  1,
@@ -113,8 +113,8 @@ func (w *wizard) deployFaucet() {
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
-	infos.node.portFull = w.readDefaultInt(infos.node.portFull)
+	fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.port)
+	infos.node.port = w.readDefaultInt(infos.node.port)
 
 	// Set a proper name to report on the stats page
 	fmt.Println()
@@ -168,7 +168,7 @@ func (w *wizard) deployFaucet() {
 		fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos, nocache); err != nil {
+	if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy faucet container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go
index 84998afc93..60aa0f7ffb 100644
--- a/cmd/puppeth/wizard_intro.go
+++ b/cmd/puppeth/wizard_intro.go
@@ -59,15 +59,16 @@ func (w *wizard) run() {
 	fmt.Println()
 
 	// Make sure we have a good network name to work with	fmt.Println()
+	// Docker accepts hyphens in image names, but doesn't like it for container names
 	if w.network == "" {
-		fmt.Println("Please specify a network name to administer (no spaces, please)")
+		fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)")
 		for {
 			w.network = w.readString()
-			if !strings.Contains(w.network, " ") {
+			if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") {
 				fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network)
 				break
 			}
-			log.Error("I also like to live dangerously, still no spaces")
+			log.Error("I also like to live dangerously, still no spaces or hyphens")
 		}
 	}
 	log.Info("Administering Ethereum network", "name", w.network)
diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go
index e19180bb16..90bf7ae3c8 100644
--- a/cmd/puppeth/wizard_netstats.go
+++ b/cmd/puppeth/wizard_netstats.go
@@ -37,8 +37,7 @@ func (w *wizard) networkStats() {
 	}
 	// Clear out some previous configs to refill from current scan
 	w.conf.ethstats = ""
-	w.conf.bootFull = w.conf.bootFull[:0]
-	w.conf.bootLight = w.conf.bootLight[:0]
+	w.conf.bootnodes = w.conf.bootnodes[:0]
 
 	// Iterate over all the specified hosts and check their status
 	var pend sync.WaitGroup
@@ -76,8 +75,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 	var (
 		genesis   string
 		ethstats  string
-		bootFull  []string
-		bootLight []string
+		bootnodes []string
 	)
 	// Ensure a valid SSH connection to the remote server
 	logger := log.New("server", server)
@@ -123,10 +121,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 		stat.services["bootnode"] = infos.Report()
 
 		genesis = string(infos.genesis)
-		bootFull = append(bootFull, infos.enodeFull)
-		if infos.enodeLight != "" {
-			bootLight = append(bootLight, infos.enodeLight)
-		}
+		bootnodes = append(bootnodes, infos.enode)
 	}
 	logger.Debug("Checking for sealnode availability")
 	if infos, err := checkNode(client, w.network, false); err != nil {
@@ -184,8 +179,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
 	if ethstats != "" {
 		w.conf.ethstats = ethstats
 	}
-	w.conf.bootFull = append(w.conf.bootFull, bootFull...)
-	w.conf.bootLight = append(w.conf.bootLight, bootLight...)
+	w.conf.bootnodes = append(w.conf.bootnodes, bootnodes...)
 
 	return stat
 }
diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go
index 097e2e41aa..a60948bc67 100644
--- a/cmd/puppeth/wizard_node.go
+++ b/cmd/puppeth/wizard_node.go
@@ -48,9 +48,9 @@ func (w *wizard) deployNode(boot bool) {
 	infos, err := checkNode(client, w.network, boot)
 	if err != nil {
 		if boot {
-			infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
+			infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256}
 		} else {
-			infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
+			infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
 		}
 	}
 	existed := err == nil
@@ -79,8 +79,8 @@ func (w *wizard) deployNode(boot bool) {
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
-	infos.portFull = w.readDefaultInt(infos.portFull)
+	fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.port)
+	infos.port = w.readDefaultInt(infos.port)
 
 	// Figure out how many peers to allow (different based on node type)
 	fmt.Println()
@@ -163,7 +163,7 @@ func (w *wizard) deployNode(boot bool) {
 		fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos, nocache); err != nil {
+	if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy Ethereum node container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go
index 7c3896a17c..933cd9ae59 100644
--- a/cmd/puppeth/wizard_wallet.go
+++ b/cmd/puppeth/wizard_wallet.go
@@ -98,7 +98,7 @@ func (w *wizard) deployWallet() {
 		fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultString("n") != "n"
 	}
-	if out, err := deployWallet(client, w.network, w.conf.bootFull, infos, nocache); err != nil {
+	if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy wallet container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)
diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go
index 29b5faefa8..adac772bab 100644
--- a/cmd/swarm/config.go
+++ b/cmd/swarm/config.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"unicode"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -97,10 +98,15 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
 	config = bzzapi.NewDefaultConfig()
 	//first load settings from config file (if provided)
 	config, err = configFileOverride(config, ctx)
+	if err != nil {
+		return nil, err
+	}
 	//override settings provided by environment variables
 	config = envVarsOverride(config)
 	//override settings provided by command line
 	config = cmdLineOverride(config, ctx)
+	//validate configuration parameters
+	err = validateConfig(config)
 
 	return
 }
@@ -194,12 +200,16 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
 	}
 
-	//EnsApi can be set to "", so can't check for empty string, as it is allowed!
 	if ctx.GlobalIsSet(EnsAPIFlag.Name) {
-		currentConfig.EnsApi = ctx.GlobalString(EnsAPIFlag.Name)
+		ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name)
+		// preserve backward compatibility to disable ENS with --ens-api=""
+		if len(ensAPIs) == 1 && ensAPIs[0] == "" {
+			ensAPIs = nil
+		}
+		currentConfig.EnsAPIs = ensAPIs
 	}
 
-	if ensaddr := ctx.GlobalString(EnsAddrFlag.Name); ensaddr != "" {
+	if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" {
 		currentConfig.EnsRoot = common.HexToAddress(ensaddr)
 	}
 
@@ -266,9 +276,8 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
 	}
 
-	//EnsApi can be set to "", so can't check for empty string, as it is allowed
-	if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists {
-		currentConfig.EnsApi = ensapi
+	if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" {
+		currentConfig.EnsAPIs = strings.Split(ensapi, ",")
 	}
 
 	if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
@@ -309,6 +318,43 @@ func checkDeprecated(ctx *cli.Context) {
 	if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" {
 		utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.")
 	}
+	// warn if --ens-api flag is set
+	if ctx.GlobalString(DeprecatedEnsAddrFlag.Name) != "" {
+		log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.")
+	}
+}
+
+//validate configuration parameters
+func validateConfig(cfg *bzzapi.Config) (err error) {
+	for _, ensAPI := range cfg.EnsAPIs {
+		if ensAPI != "" {
+			if err := validateEnsAPIs(ensAPI); err != nil {
+				return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err)
+			}
+		}
+	}
+	return nil
+}
+
+//validate EnsAPIs configuration parameter
+func validateEnsAPIs(s string) (err error) {
+	// missing contract address
+	if strings.HasPrefix(s, "@") {
+		return errors.New("missing contract address")
+	}
+	// missing url
+	if strings.HasSuffix(s, "@") {
+		return errors.New("missing url")
+	}
+	// missing tld
+	if strings.HasPrefix(s, ":") {
+		return errors.New("missing tld")
+	}
+	// missing url
+	if strings.HasSuffix(s, ":") {
+		return errors.New("missing url")
+	}
+	return nil
 }
 
 //print a Config as string
diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go
index 166980d148..9bf584f50c 100644
--- a/cmd/swarm/config_test.go
+++ b/cmd/swarm/config_test.go
@@ -457,3 +457,98 @@ func TestCmdLineOverridesFile(t *testing.T) {
 
 	node.Shutdown()
 }
+
+func TestValidateConfig(t *testing.T) {
+	for _, c := range []struct {
+		cfg *api.Config
+		err string
+	}{
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				":314159265dD8dbb310642f98f50C066173C1259",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"@/data/testnet/geth.ipc",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/geth.ipc\": missing contract address",
+		},
+	} {
+		err := validateConfig(c.cfg)
+		if c.err != "" && err.Error() != c.err {
+			t.Errorf("expected error %q, got %q", c.err, err)
+		}
+		if c.err == "" && err != nil {
+			t.Errorf("unexpected error %q", err)
+		}
+	}
+}
diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go
index 77315a4265..360020b77b 100644
--- a/cmd/swarm/main.go
+++ b/cmd/swarm/main.go
@@ -17,11 +17,9 @@
 package main
 
 import (
-	"context"
 	"crypto/ecdsa"
 	"fmt"
 	"io/ioutil"
-	"math/big"
 	"os"
 	"os/signal"
 	"runtime"
@@ -29,14 +27,12 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
-	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
@@ -45,9 +41,9 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
+	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
 
 	"gopkg.in/urfave/cli.v1"
 )
@@ -110,16 +106,11 @@ var (
 		Usage:  "Swarm Syncing enabled (default true)",
 		EnvVar: SWARM_ENV_SYNC_ENABLE,
 	}
-	EnsAPIFlag = cli.StringFlag{
+	EnsAPIFlag = cli.StringSliceFlag{
 		Name:   "ens-api",
-		Usage:  "URL of the Ethereum API provider to use for ENS record lookups",
+		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
 		EnvVar: SWARM_ENV_ENS_API,
 	}
-	EnsAddrFlag = cli.StringFlag{
-		Name:   "ens-addr",
-		Usage:  "ENS contract address (default is detected as testnet or mainnet using --ens-api)",
-		EnvVar: SWARM_ENV_ENS_ADDR,
-	}
 	SwarmApiFlag = cli.StringFlag{
 		Name:  "bzzapi",
 		Usage: "Swarm HTTP endpoint",
@@ -156,6 +147,10 @@ var (
 		Name:  "ethapi",
 		Usage: "DEPRECATED: please use --ens-api and --swap-api",
 	}
+	DeprecatedEnsAddrFlag = cli.StringFlag{
+		Name:  "ens-addr",
+		Usage: "DEPRECATED: ENS contract address, please use --ens-api with contract address according to its format",
+	}
 )
 
 //declare a few constant error messages, useful for later error check comparisons in test
@@ -343,7 +338,6 @@ DEPRECATED: use 'swarm db clean'.
 		// bzzd-specific flags
 		CorsStringFlag,
 		EnsAPIFlag,
-		EnsAddrFlag,
 		SwarmTomlConfigPathFlag,
 		SwarmConfigPathFlag,
 		SwarmSwapEnabledFlag,
@@ -363,11 +357,17 @@ DEPRECATED: use 'swarm db clean'.
 		SwarmUploadMimeType,
 		//deprecated flags
 		DeprecatedEthAPIFlag,
+		DeprecatedEnsAddrFlag,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
+	app.Flags = append(app.Flags, swarmmetrics.Flags...)
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())
-		return debug.Setup(ctx)
+		if err := debug.Setup(ctx); err != nil {
+			return err
+		}
+		swarmmetrics.Setup(ctx)
+		return nil
 	}
 	app.After = func(ctx *cli.Context) error {
 		debug.Exit()
@@ -448,38 +448,6 @@ func bzzd(ctx *cli.Context) error {
 	return nil
 }
 
-// detectEnsAddr determines the ENS contract address by getting both the
-// version and genesis hash using the client and matching them to either
-// mainnet or testnet addresses
-func detectEnsAddr(client *rpc.Client) (common.Address, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	var version string
-	if err := client.CallContext(ctx, &version, "net_version"); err != nil {
-		return common.Address{}, err
-	}
-
-	block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0))
-	if err != nil {
-		return common.Address{}, err
-	}
-
-	switch {
-
-	case version == "1" && block.Hash() == params.MainnetGenesisHash:
-		log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress)
-		return ens.MainNetAddress, nil
-
-	case version == "3" && block.Hash() == params.TestnetGenesisHash:
-		log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress)
-		return ens.TestNetAddress, nil
-
-	default:
-		return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash())
-	}
-}
-
 func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.Node) {
 
 	//define the swarm service boot function
@@ -494,27 +462,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 			}
 		}
 
-		var ensClient *ethclient.Client
-		if bzzconfig.EnsApi != "" {
-			log.Info("connecting to ENS API", "url", bzzconfig.EnsApi)
-			client, err := rpc.Dial(bzzconfig.EnsApi)
-			if err != nil {
-				return nil, fmt.Errorf("error connecting to ENS API %s: %s", bzzconfig.EnsApi, err)
-			}
-			ensClient = ethclient.NewClient(client)
-
-			//no ENS root address set yet
-			if bzzconfig.EnsRoot == (common.Address{}) {
-				ensAddr, err := detectEnsAddr(client)
-				if err == nil {
-					bzzconfig.EnsRoot = ensAddr
-				} else {
-					log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err)
-				}
-			}
-		}
-
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors)
+		return swarm.NewSwarm(ctx, swapClient, bzzconfig)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {
diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go
index ed15028685..594cfa55cb 100644
--- a/cmd/swarm/run_test.go
+++ b/cmd/swarm/run_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go
index 5656186e1c..df7fc216af 100644
--- a/cmd/swarm/upload_test.go
+++ b/cmd/swarm/upload_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go
index 53cdf7861c..186d18d8f5 100644
--- a/cmd/utils/cmd.go
+++ b/cmd/utils/cmd.go
@@ -25,6 +25,7 @@ import (
 	"os/signal"
 	"runtime"
 	"strings"
+	"syscall"
 
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -64,7 +65,7 @@ func StartNode(stack *node.Node) {
 	}
 	go func() {
 		sigc := make(chan os.Signal, 1)
-		signal.Notify(sigc, os.Interrupt)
+		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
 		defer signal.Stop(sigc)
 		<-sigc
 		log.Info("Got interrupt, shutting down...")
@@ -85,7 +86,7 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 	// If a signal is received, the import will stop at the next batch.
 	interrupt := make(chan os.Signal, 1)
 	stop := make(chan struct{})
-	signal.Notify(interrupt, os.Interrupt)
+	signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
 	defer signal.Stop(interrupt)
 	defer close(interrupt)
 	go func() {
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 2a2909ff2c..5fd5013f00 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -397,6 +397,11 @@ var (
 		Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
 		Value: "",
 	}
+	RPCVirtualHostsFlag = cli.StringFlag{
+		Name:  "rpcvhosts",
+		Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.",
+		Value: "localhost",
+	}
 	RPCApiFlag = cli.StringFlag{
 		Name:  "rpcapi",
 		Usage: "API's offered over the HTTP-RPC interface",
@@ -690,6 +695,8 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) {
 	if ctx.GlobalIsSet(RPCApiFlag.Name) {
 		cfg.HTTPModules = splitAndTrim(ctx.GlobalString(RPCApiFlag.Name))
 	}
+
+	cfg.HTTPVirtualHosts = splitAndTrim(ctx.GlobalString(RPCVirtualHostsFlag.Name))
 }
 
 // setWS creates the WebSocket RPC listener interface string from the set
diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go
index e69b57d69f..f8606bf824 100644
--- a/cmd/wnode/main.go
+++ b/cmd/wnode/main.go
@@ -43,7 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/whisper/mailserver"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	"golang.org/x/crypto/pbkdf2"
 )
 
@@ -61,27 +61,29 @@ var (
 
 // encryption
 var (
-	symKey     []byte
-	pub        *ecdsa.PublicKey
-	asymKey    *ecdsa.PrivateKey
-	nodeid     *ecdsa.PrivateKey
-	topic      whisper.TopicType
-	asymKeyID  string
-	filterID   string
-	symPass    string
-	msPassword string
+	symKey  []byte
+	pub     *ecdsa.PublicKey
+	asymKey *ecdsa.PrivateKey
+	nodeid  *ecdsa.PrivateKey
+	topic   whisper.TopicType
+
+	asymKeyID    string
+	asymFilterID string
+	symFilterID  string
+	symPass      string
+	msPassword   string
 )
 
 // cmd arguments
 var (
-	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
-	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't initiate connection to peers, just wait for incoming connections")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither encrypt nor decrypt messages")
 	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
 	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
 	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
 	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
 	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
-	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)")
 	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
 	argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level")
@@ -97,7 +99,7 @@ var (
 	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
 	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
 	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
-	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
+	argSaveDir = flag.String("savedir", "", "directory where all incoming messages will be saved as files")
 )
 
 func main() {
@@ -263,7 +265,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("wnode", "5.0"),
+			Name:           common.MakeName("wnode", "6.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -363,13 +365,22 @@ func configureNode() {
 		}
 	}
 
-	filter := whisper.Filter{
+	symFilter := whisper.Filter{
 		KeySym:   symKey,
+		Topics:   [][]byte{topic[:]},
+		AllowP2P: p2pAccept,
+	}
+	symFilterID, err = shh.Subscribe(&symFilter)
+	if err != nil {
+		utils.Fatalf("Failed to install filter: %s", err)
+	}
+
+	asymFilter := whisper.Filter{
 		KeyAsym:  asymKey,
 		Topics:   [][]byte{topic[:]},
 		AllowP2P: p2pAccept,
 	}
-	filterID, err = shh.Subscribe(&filter)
+	asymFilterID, err = shh.Subscribe(&asymFilter)
 	if err != nil {
 		utils.Fatalf("Failed to install filter: %s", err)
 	}
@@ -522,9 +533,14 @@ func sendMsg(payload []byte) common.Hash {
 }
 
 func messageLoop() {
-	f := shh.GetFilter(filterID)
-	if f == nil {
-		utils.Fatalf("filter is not installed")
+	sf := shh.GetFilter(symFilterID)
+	if sf == nil {
+		utils.Fatalf("symmetric filter is not installed")
+	}
+
+	af := shh.GetFilter(asymFilterID)
+	if af == nil {
+		utils.Fatalf("asymmetric filter is not installed")
 	}
 
 	ticker := time.NewTicker(time.Millisecond * 50)
@@ -532,11 +548,18 @@ func messageLoop() {
 	for {
 		select {
 		case <-ticker.C:
-			messages := f.Retrieve()
+			m1 := sf.Retrieve()
+			m2 := af.Retrieve()
+			messages := append(m1, m2...)
 			for _, msg := range messages {
-				if *fileExMode || len(msg.Payload) > 2048 {
+				// All messages are saved upon specifying argSaveDir.
+				// fileExMode only specifies how messages are displayed on the console after they are saved.
+				// if fileExMode == true, only the hashes are displayed, since messages might be too big.
+				if len(*argSaveDir) > 0 {
 					writeMessageToFile(*argSaveDir, msg)
-				} else {
+				}
+
+				if !*fileExMode && len(msg.Payload) <= 2048 {
 					printMessageInfo(msg)
 				}
 			}
@@ -571,19 +594,22 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
 		address = crypto.PubkeyToAddress(*msg.Src)
 	}
 
-	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
-		// message from myself: don't save, only report
-		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
-	} else if len(dir) > 0 {
+	// this is a sample code; uncomment if you don't want to save your own messages.
+	//if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+	//	fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name)
+	//	return
+	//}
+
+	if len(dir) > 0 {
 		fullpath := filepath.Join(dir, name)
-		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		err := ioutil.WriteFile(fullpath, msg.Raw, 0644)
 		if err != nil {
 			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
 		} else {
-			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw))
 		}
 	} else {
-		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+		fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name)
 	}
 }
 
@@ -631,7 +657,7 @@ func requestExpiredMessagesLoop() {
 		params.PoW = *argServerPoW
 		params.Payload = data
 		params.KeySym = key
-		params.Src = nodeid
+		params.Src = asymKey
 		params.WorkTime = 5
 
 		msg, err := whisper.NewSentMessage(¶ms)
diff --git a/common/big.go b/common/big.go
index b552608bc7..65d4377bf7 100644
--- a/common/big.go
+++ b/common/big.go
@@ -25,6 +25,6 @@ var (
 	Big3   = big.NewInt(3)
 	Big0   = big.NewInt(0)
 	Big32  = big.NewInt(32)
-	Big256 = big.NewInt(0xff)
+	Big256 = big.NewInt(256)
 	Big257 = big.NewInt(257)
 )
diff --git a/common/fdlimit/fdlimit_freebsd.go b/common/fdlimit/fdlimit_freebsd.go
index 25caaafe21..c126b0c265 100644
--- a/common/fdlimit/fdlimit_freebsd.go
+++ b/common/fdlimit/fdlimit_freebsd.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 // +build freebsd
 
diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go
index 05e9f0b658..a9ee9ab36a 100644
--- a/common/fdlimit/fdlimit_test.go
+++ b/common/fdlimit/fdlimit_test.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package fdlimit
 
diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go
index 27c7e783f7..a258132353 100644
--- a/common/fdlimit/fdlimit_unix.go
+++ b/common/fdlimit/fdlimit_unix.go
@@ -1,18 +1,18 @@
 // Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 // +build linux darwin netbsd openbsd solaris
 
diff --git a/common/fdlimit/fdlimit_windows.go b/common/fdlimit/fdlimit_windows.go
index efcd3220ea..863c58bedf 100644
--- a/common/fdlimit/fdlimit_windows.go
+++ b/common/fdlimit/fdlimit_windows.go
@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package fdlimit
 
diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go
index 10767bb312..905a7b1ea7 100644
--- a/consensus/ethash/algorithm.go
+++ b/consensus/ethash/algorithm.go
@@ -19,6 +19,7 @@ package ethash
 import (
 	"encoding/binary"
 	"hash"
+	"math/big"
 	"reflect"
 	"runtime"
 	"sync"
@@ -47,6 +48,48 @@ const (
 	loopAccesses       = 64      // Number of accesses in hashimoto loop
 )
 
+// cacheSize returns the size of the ethash verification cache that belongs to a certain
+// block number.
+func cacheSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return cacheSizes[epoch]
+	}
+	return calcCacheSize(epoch)
+}
+
+// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcCacheSize(epoch int) uint64 {
+	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
+	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * hashBytes
+	}
+	return size
+}
+
+// datasetSize returns the size of the ethash mining dataset that belongs to a certain
+// block number.
+func datasetSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return datasetSizes[epoch]
+	}
+	return calcDatasetSize(epoch)
+}
+
+// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcDatasetSize(epoch int) uint64 {
+	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
+	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * mixBytes
+	}
+	return size
+}
+
 // hasher is a repetitive hasher allowing the same hash data structures to be
 // reused between hash runs instead of requiring new ones to be created.
 type hasher func(dest []byte, data []byte)
diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go
deleted file mode 100644
index c7f7f48e41..0000000000
--- a/consensus/ethash/algorithm_go1.7.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build !go1.8
-
-package ethash
-
-// cacheSize calculates and returns the size of the ethash verification cache that
-// belongs to a certain block number. The cache size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func cacheSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}
-
-// datasetSize calculates and returns the size of the ethash mining dataset that
-// belongs to a certain block number. The dataset size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func datasetSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}
diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go
deleted file mode 100644
index 975fdffe51..0000000000
--- a/consensus/ethash/algorithm_go1.8.go
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build go1.8
-
-package ethash
-
-import "math/big"
-
-// cacheSize returns the size of the ethash verification cache that belongs to a certain
-// block number.
-func cacheSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	return calcCacheSize(epoch)
-}
-
-// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcCacheSize(epoch int) uint64 {
-	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
-	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * hashBytes
-	}
-	return size
-}
-
-// datasetSize returns the size of the ethash mining dataset that belongs to a certain
-// block number.
-func datasetSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	return calcDatasetSize(epoch)
-}
-
-// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcDatasetSize(epoch int) uint64 {
-	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
-	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * mixBytes
-	}
-	return size
-}
diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go
deleted file mode 100644
index 6648bd6a97..0000000000
--- a/consensus/ethash/algorithm_go1.8_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// +build go1.8
-
-package ethash
-
-import "testing"
-
-// Tests whether the dataset size calculator works correctly by cross checking the
-// hard coded lookup table with the value generated by it.
-func TestSizeCalculations(t *testing.T) {
-	// Verify all the cache and dataset sizes from the lookup table.
-	for epoch, want := range cacheSizes {
-		if size := calcCacheSize(epoch); size != want {
-			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-	for epoch, want := range datasetSizes {
-		if size := calcDatasetSize(epoch); size != want {
-			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-}
diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go
index a54f3b582c..841e39233f 100644
--- a/consensus/ethash/algorithm_test.go
+++ b/consensus/ethash/algorithm_test.go
@@ -30,6 +30,22 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
+// Tests whether the dataset size calculator works correctly by cross checking the
+// hard coded lookup table with the value generated by it.
+func TestSizeCalculations(t *testing.T) {
+	// Verify all the cache and dataset sizes from the lookup table.
+	for epoch, want := range cacheSizes {
+		if size := calcCacheSize(epoch); size != want {
+			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+	for epoch, want := range datasetSizes {
+		if size := calcDatasetSize(epoch); size != want {
+			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+}
+
 // Tests that verification caches can be correctly generated.
 func TestCacheGeneration(t *testing.T) {
 	tests := []struct {
diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go
index 92a23d4a4d..3f860bf473 100644
--- a/consensus/ethash/consensus.go
+++ b/consensus/ethash/consensus.go
@@ -53,7 +53,6 @@ var (
 	errDuplicateUncle    = errors.New("duplicate uncle")
 	errUncleIsAncestor   = errors.New("uncle is ancestor")
 	errDanglingUncle     = errors.New("uncle's parent is not ancestor")
-	errNonceOutOfRange   = errors.New("nonce out of range")
 	errInvalidDifficulty = errors.New("non-positive difficulty")
 	errInvalidMixDigest  = errors.New("invalid mix digest")
 	errInvalidPoW        = errors.New("invalid proof-of-work")
@@ -474,18 +473,13 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	if ethash.shared != nil {
 		return ethash.shared.VerifySeal(chain, header)
 	}
-	// Sanity check that the block number is below the lookup table size (60M blocks)
-	number := header.Number.Uint64()
-	if number/epochLength >= maxEpoch {
-		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
-		return errNonceOutOfRange
-	}
 	// Ensure that we have a valid difficulty for the block
 	if header.Difficulty.Sign() <= 0 {
 		return errInvalidDifficulty
 	}
-
 	// Recompute the digest and PoW value and verify against the header
+	number := header.Number.Uint64()
+
 	cache := ethash.cache(number)
 	size := datasetSize(number)
 	if ethash.config.PowMode == ModeTest {
diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go
index 91e20112ae..1b3dcee302 100644
--- a/consensus/ethash/ethash.go
+++ b/consensus/ethash/ethash.go
@@ -35,9 +35,9 @@ import (
 	mmap "github.com/edsrzf/mmap-go"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/hashicorp/golang-lru/simplelru"
-	metrics "github.com/rcrowley/go-metrics"
 )
 
 var ErrInvalidDumpMagic = errors.New("invalid dump magic")
diff --git a/console/console.go b/console/console.go
index 52fe1f542a..b280d4e65d 100644
--- a/console/console.go
+++ b/console/console.go
@@ -26,6 +26,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"syscall"
 
 	"github.com/ethereum/go-ethereum/internal/jsre"
 	"github.com/ethereum/go-ethereum/internal/web3ext"
@@ -332,7 +333,7 @@ func (c *Console) Interactive() {
 	}()
 	// Monitor Ctrl-C too in case the input is empty and we need to bail
 	abort := make(chan os.Signal, 1)
-	signal.Notify(abort, os.Interrupt)
+	signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
 
 	// Start sending prompts to the user and reading back inputs
 	for {
diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile
index c7b71c7263..8d4e7fe81a 100644
--- a/containers/docker/master-alpine/Dockerfile
+++ b/containers/docker/master-alpine/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine:3.7
 
 RUN \
   apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
-  git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
+  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \
   apk del go git make gcc musl-dev linux-headers && \
diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile
index bba70abfdd..4cfc4f58c0 100644
--- a/containers/docker/master-ubuntu/Dockerfile
+++ b/containers/docker/master-ubuntu/Dockerfile
@@ -5,7 +5,7 @@ ENV PATH=/usr/lib/go-1.9/bin:$PATH
 RUN \
   apt-get update && apt-get upgrade -q -y && \
   apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \
-  git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
+  git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
   (cd go-ethereum && make geth) && \
   cp go-ethereum/build/bin/geth /geth && \
   apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \
diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go
index b7555d0815..6b6b28e657 100644
--- a/contracts/chequebook/cheque_test.go
+++ b/contracts/chequebook/cheque_test.go
@@ -281,8 +281,8 @@ func TestDeposit(t *testing.T) {
 		t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
 	}
 
-	// autodeposit every 30ms if new cheque issued
-	interval := 30 * time.Millisecond
+	// autodeposit every 200ms if new cheque issued
+	interval := 200 * time.Millisecond
 	chbook.AutoDeposit(interval, common.Big1, balance)
 	_, err = chbook.Issue(addr1, amount)
 	if err != nil {
diff --git a/core/asm/lexer.go b/core/asm/lexer.go
index a34b2cbd8f..4054999504 100644
--- a/core/asm/lexer.go
+++ b/core/asm/lexer.go
@@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn {
 			return lexComment
 		case isSpace(r):
 			l.ignore()
-		case isAlphaNumeric(r) || r == '_':
+		case isLetter(r) || r == '_':
 			return lexElement
 		case isNumber(r):
 			return lexNumber
@@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn {
 	return lexLine
 }
 
-func isAlphaNumeric(t rune) bool {
+func isLetter(t rune) bool {
 	return unicode.IsLetter(t)
 }
 
diff --git a/core/blockchain.go b/core/blockchain.go
index 8d141fddb5..6006e66740 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -46,7 +46,7 @@ import (
 )
 
 var (
-	blockInsertTimer = metrics.NewTimer("chain/inserts")
+	blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
 
 	ErrNoGenesis = errors.New("Genesis not found in chain")
 )
@@ -107,8 +107,8 @@ type BlockChain struct {
 	procmu  sync.RWMutex // block processor lock
 
 	checkpoint       int          // checkpoint counts towards the new checkpoint
-	currentBlock     *types.Block // Current head of the block chain
-	currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
+	currentBlock     atomic.Value // Current head of the block chain
+	currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 
 	stateCache   state.Database // State database to reuse between imports (contains state cache)
 	bodyCache    *lru.Cache     // Cache for the most recent block bodies
@@ -224,10 +224,10 @@ func (bc *BlockChain) loadLastState() error {
 		}
 	}
 	// Everything seems to be fine, set as the head block
-	bc.currentBlock = currentBlock
+	bc.currentBlock.Store(currentBlock)
 
 	// Restore the last known head header
-	currentHeader := bc.currentBlock.Header()
+	currentHeader := currentBlock.Header()
 	if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) {
 		if header := bc.GetHeaderByHash(head); header != nil {
 			currentHeader = header
@@ -236,21 +236,23 @@ func (bc *BlockChain) loadLastState() error {
 	bc.hc.SetCurrentHeader(currentHeader)
 
 	// Restore the last known head fast block
-	bc.currentFastBlock = bc.currentBlock
+	bc.currentFastBlock.Store(currentBlock)
 	if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) {
 		if block := bc.GetBlockByHash(head); block != nil {
-			bc.currentFastBlock = block
+			bc.currentFastBlock.Store(block)
 		}
 	}
 
 	// Issue a status log for the user
+	currentFastBlock := bc.CurrentFastBlock()
+
 	headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64())
-	blockTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
-	fastTd := bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64())
+	blockTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
+	fastTd := bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64())
 
 	log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd)
-	log.Info("Loaded most recent local full block", "number", bc.currentBlock.Number(), "hash", bc.currentBlock.Hash(), "td", blockTd)
-	log.Info("Loaded most recent local fast block", "number", bc.currentFastBlock.Number(), "hash", bc.currentFastBlock.Hash(), "td", fastTd)
+	log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd)
+	log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd)
 
 	return nil
 }
@@ -279,30 +281,32 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	bc.futureBlocks.Purge()
 
 	// Rewind the block chain, ensuring we don't end up with a stateless head block
-	if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() {
-		bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())
+	if currentBlock := bc.CurrentBlock(); currentBlock != nil && currentHeader.Number.Uint64() < currentBlock.NumberU64() {
+		bc.currentBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()))
 	}
-	if bc.currentBlock != nil {
-		if _, err := state.New(bc.currentBlock.Root(), bc.stateCache); err != nil {
+	if currentBlock := bc.CurrentBlock(); currentBlock != nil {
+		if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
 			// Rewound state missing, rolled back to before pivot, reset to genesis
-			bc.currentBlock = nil
+			bc.currentBlock.Store(bc.genesisBlock)
 		}
 	}
 	// Rewind the fast block in a simpleton way to the target head
-	if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() {
-		bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())
+	if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock != nil && currentHeader.Number.Uint64() < currentFastBlock.NumberU64() {
+		bc.currentFastBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()))
 	}
 	// If either blocks reached nil, reset to the genesis state
-	if bc.currentBlock == nil {
-		bc.currentBlock = bc.genesisBlock
+	if currentBlock := bc.CurrentBlock(); currentBlock == nil {
+		bc.currentBlock.Store(bc.genesisBlock)
 	}
-	if bc.currentFastBlock == nil {
-		bc.currentFastBlock = bc.genesisBlock
+	if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock == nil {
+		bc.currentFastBlock.Store(bc.genesisBlock)
 	}
-	if err := WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()); err != nil {
+	currentBlock := bc.CurrentBlock()
+	currentFastBlock := bc.CurrentFastBlock()
+	if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head full block", "err", err)
 	}
-	if err := WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()); err != nil {
+	if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil {
 		log.Crit("Failed to reset head fast block", "err", err)
 	}
 	return bc.loadLastState()
@@ -321,7 +325,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	}
 	// If all checks out, manually set the head block
 	bc.mu.Lock()
-	bc.currentBlock = block
+	bc.currentBlock.Store(block)
 	bc.mu.Unlock()
 
 	log.Info("Committed new head block", "number", block.Number(), "hash", hash)
@@ -330,28 +334,19 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 
 // GasLimit returns the gas limit of the current HEAD block.
 func (bc *BlockChain) GasLimit() uint64 {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentBlock.GasLimit()
+	return bc.CurrentBlock().GasLimit()
 }
 
 // CurrentBlock retrieves the current head block of the canonical chain. The
 // block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentBlock() *types.Block {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentBlock
+	return bc.currentBlock.Load().(*types.Block)
 }
 
 // CurrentFastBlock retrieves the current fast-sync head block of the canonical
 // chain. The block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentFastBlock() *types.Block {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
-	return bc.currentFastBlock
+	return bc.currentFastBlock.Load().(*types.Block)
 }
 
 // SetProcessor sets the processor required for making state modifications.
@@ -416,10 +411,10 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	}
 	bc.genesisBlock = genesis
 	bc.insert(bc.genesisBlock)
-	bc.currentBlock = bc.genesisBlock
+	bc.currentBlock.Store(bc.genesisBlock)
 	bc.hc.SetGenesis(bc.genesisBlock.Header())
 	bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
-	bc.currentFastBlock = bc.genesisBlock
+	bc.currentFastBlock.Store(bc.genesisBlock)
 
 	return nil
 }
@@ -444,7 +439,7 @@ func (bc *BlockChain) repair(head **types.Block) error {
 
 // Export writes the active chain to the given writer.
 func (bc *BlockChain) Export(w io.Writer) error {
-	return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64())
+	return bc.ExportN(w, uint64(0), bc.CurrentBlock().NumberU64())
 }
 
 // ExportN writes a subset of the active chain to the given writer.
@@ -488,7 +483,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 	if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil {
 		log.Crit("Failed to insert head block hash", "err", err)
 	}
-	bc.currentBlock = block
+	bc.currentBlock.Store(block)
 
 	// If the block is better than our head or is on a different chain, force update heads
 	if updateHeads {
@@ -497,7 +492,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 		if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil {
 			log.Crit("Failed to insert head fast block hash", "err", err)
 		}
-		bc.currentFastBlock = block
+		bc.currentFastBlock.Store(block)
 	}
 }
 
@@ -648,22 +643,21 @@ func (bc *BlockChain) Stop() {
 	bc.wg.Wait()
 
 	// Ensure the state of a recent block is also stored to disk before exiting.
-	// It is fine if this state does not exist (fast start/stop cycle), but it is
-	// advisable to leave an N block gap from the head so 1) a restart loads up
-	// the last N blocks as sync assistance to remote nodes; 2) a restart during
-	// a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from
-	// missing states are constantly tested.
-	//
-	// This may be tuned a bit on mainnet if its too annoying to reprocess the last
-	// N blocks.
+	// We're writing three different states to catch different restart scenarios:
+	//  - HEAD:     So we don't need to reprocess any blocks in the general case
+	//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
+	//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
 	if !bc.cacheConfig.Disabled {
 		triedb := bc.stateCache.TrieDB()
-		if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory {
-			recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1)
 
-			log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
-			if err := triedb.Commit(recent.Root(), true); err != nil {
-				log.Error("Failed to commit recent state trie", "err", err)
+		for _, offset := range []uint64{0, 1, triesInMemory - 1} {
+			if number := bc.CurrentBlock().NumberU64(); number > offset {
+				recent := bc.GetBlockByNumber(number - offset)
+
+				log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+				if err := triedb.Commit(recent.Root(), true); err != nil {
+					log.Error("Failed to commit recent state trie", "err", err)
+				}
 			}
 		}
 		for !bc.triegc.Empty() {
@@ -715,13 +709,15 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
 		if currentHeader.Hash() == hash {
 			bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1))
 		}
-		if bc.currentFastBlock.Hash() == hash {
-			bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1)
-			WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash())
+		if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash {
+			newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1)
+			bc.currentFastBlock.Store(newFastBlock)
+			WriteHeadFastBlockHash(bc.db, newFastBlock.Hash())
 		}
-		if bc.currentBlock.Hash() == hash {
-			bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1)
-			WriteHeadBlockHash(bc.db, bc.currentBlock.Hash())
+		if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash {
+			newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1)
+			bc.currentBlock.Store(newBlock)
+			WriteHeadBlockHash(bc.db, newBlock.Hash())
 		}
 	}
 }
@@ -830,11 +826,12 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	bc.mu.Lock()
 	head := blockChain[len(blockChain)-1]
 	if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case
-		if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 {
+		currentFastBlock := bc.CurrentFastBlock()
+		if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 {
 			if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil {
 				log.Crit("Failed to update head fast block hash", "err", err)
 			}
-			bc.currentFastBlock = head
+			bc.currentFastBlock.Store(head)
 		}
 	}
 	bc.mu.Unlock()
@@ -881,7 +878,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	bc.mu.Lock()
 	defer bc.mu.Unlock()
 
-	localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
+	currentBlock := bc.CurrentBlock()
+	localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
 	externTd := new(big.Int).Add(block.Difficulty(), ptd)
 
 	// Irrelevant of the canonical status, write the block itself to the database
@@ -926,13 +924,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 				if chosen < lastWrite+triesInMemory {
 					switch {
 					case size >= 2*limit:
-						log.Error("Trie memory critical, forcing to disk", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+						log.Warn("State memory usage too high, committing", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
 					case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit:
-						log.Error("Trie timing critical, forcing to disk", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
-					case size > limit:
-						log.Warn("Trie memory at dangerous levels", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
-					case bc.gcproc > bc.cacheConfig.TrieTimeLimit:
-						log.Warn("Trie timing at dangerous levels", "time", bc.gcproc, "limit", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
+						log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
 					}
 				}
 				// If optimum or critical limits reached, write to disk
@@ -960,14 +954,15 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	// Second clause in the if statement reduces the vulnerability to selfish mining.
 	// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
 	reorg := externTd.Cmp(localTd) > 0
+	currentBlock = bc.CurrentBlock()
 	if !reorg && externTd.Cmp(localTd) == 0 {
 		// Split same-difficulty blocks by number, then at random
-		reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5)
+		reorg = block.NumberU64() < currentBlock.NumberU64() || (block.NumberU64() == currentBlock.NumberU64() && mrand.Float64() < 0.5)
 	}
 	if reorg {
 		// Reorganise the chain if the parent is not the head block
-		if block.ParentHash() != bc.currentBlock.Hash() {
-			if err := bc.reorg(bc.currentBlock, block); err != nil {
+		if block.ParentHash() != currentBlock.Hash() {
+			if err := bc.reorg(currentBlock, block); err != nil {
 				return NonStatTy, err
 			}
 		}
@@ -1070,8 +1065,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		}
 		switch {
 		case err == ErrKnownBlock:
-			stats.ignored++
-			continue
+			// Block and state both already known. However if the current block is below
+			// this number we did a rollback and we should reimport it nonetheless.
+			if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
+				stats.ignored++
+				continue
+			}
 
 		case err == consensus.ErrFutureBlock:
 			// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
@@ -1092,7 +1091,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		case err == consensus.ErrPrunedAncestor:
 			// Block competing with the canonical chain, store in the db, but don't process
 			// until the competitor TD goes above the canonical TD
-			localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
+			currentBlock := bc.CurrentBlock()
+			localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
 			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
 			if localTd.Cmp(externTd) > 0 {
 				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
@@ -1481,9 +1481,6 @@ func (bc *BlockChain) writeHeader(header *types.Header) error {
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (bc *BlockChain) CurrentHeader() *types.Header {
-	bc.mu.RLock()
-	defer bc.mu.RUnlock()
-
 	return bc.hc.CurrentHeader()
 }
 
diff --git a/core/blockchain_test.go b/core/blockchain_test.go
index 635379161c..748cdc5c71 100644
--- a/core/blockchain_test.go
+++ b/core/blockchain_test.go
@@ -34,26 +34,6 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
-// newTestBlockChain creates a blockchain without validation.
-func newTestBlockChain(fake bool) *BlockChain {
-	db, _ := ethdb.NewMemDatabase()
-	gspec := &Genesis{
-		Config:     params.TestChainConfig,
-		Difficulty: big.NewInt(1),
-	}
-	gspec.MustCommit(db)
-	engine := ethash.NewFullFaker()
-	if !fake {
-		engine = ethash.NewTester()
-	}
-	blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
-	if err != nil {
-		panic(err)
-	}
-	blockchain.SetValidator(bproc{})
-	return blockchain
-}
-
 // Test fork of length N starting from block i
 func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) {
 	// Copy old chain up to #i into a new db
@@ -183,13 +163,18 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *
 }
 
 func TestLastBlock(t *testing.T) {
-	bchain := newTestBlockChain(false)
-	defer bchain.Stop()
+	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
-	block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0]
-	bchain.insert(block)
-	if block.Hash() != GetHeadBlockHash(bchain.db) {
-		t.Errorf("Write/Get HeadBlockHash failed")
+	blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0)
+	if _, err := blockchain.InsertChain(blocks); err != nil {
+		t.Fatalf("Failed to insert block: %v", err)
+	}
+	if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) {
+		t.Fatalf("Write/Get HeadBlockHash failed")
 	}
 }
 
@@ -337,55 +322,13 @@ func testBrokenChain(t *testing.T, full bool) {
 	}
 }
 
-type bproc struct{}
-
-func (bproc) ValidateBody(*types.Block) error { return nil }
-func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error {
-	return nil
-}
-func (bproc) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) {
-	return nil, nil, 0, nil
-}
-
-func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header {
-	blocks := makeBlockChainWithDiff(genesis, d, seed)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	return headers
-}
-
-func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
-	var chain []*types.Block
-	for i, difficulty := range d {
-		header := &types.Header{
-			Coinbase:    common.Address{seed},
-			Number:      big.NewInt(int64(i + 1)),
-			Difficulty:  big.NewInt(int64(difficulty)),
-			UncleHash:   types.EmptyUncleHash,
-			TxHash:      types.EmptyRootHash,
-			ReceiptHash: types.EmptyRootHash,
-			Time:        big.NewInt(int64(i) + 1),
-		}
-		if i == 0 {
-			header.ParentHash = genesis.Hash()
-		} else {
-			header.ParentHash = chain[i-1].Hash()
-		}
-		block := types.NewBlockWithHeader(header)
-		chain = append(chain, block)
-	}
-	return chain
-}
-
 // Tests that reorganising a long difficult chain after a short easy one
 // overwrites the canonical numbers and links in the database.
 func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) }
 func TestReorgLongBlocks(t *testing.T)  { testReorgLong(t, true) }
 
 func testReorgLong(t *testing.T, full bool) {
-	testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full)
+	testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280, full)
 }
 
 // Tests that reorganising a short difficult chain after a long easy one
@@ -394,45 +337,82 @@ func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) }
 func TestReorgShortBlocks(t *testing.T)  { testReorgShort(t, true) }
 
 func testReorgShort(t *testing.T, full bool) {
-	testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full)
+	// Create a long easy chain vs. a short heavy one. Due to difficulty adjustment
+	// we need a fairly long chain of blocks with different difficulties for a short
+	// one to become heavyer than a long one. The 96 is an empirical value.
+	easy := make([]int64, 96)
+	for i := 0; i < len(easy); i++ {
+		easy[i] = 60
+	}
+	diff := make([]int64, len(easy)-1)
+	for i := 0; i < len(diff); i++ {
+		diff[i] = -9
+	}
+	testReorg(t, easy, diff, 12615120, full)
 }
 
-func testReorg(t *testing.T, first, second []int, td int64, full bool) {
-	bc := newTestBlockChain(true)
-	defer bc.Stop()
+func testReorg(t *testing.T, first, second []int64, td int64, full bool) {
+	// Create a pristine chain and database
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
 	// Insert an easy and a difficult chain afterwards
+	easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) {
+		b.OffsetTime(first[i])
+	})
+	diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) {
+		b.OffsetTime(second[i])
+	})
 	if full {
-		bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, first, 11))
-		bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, second, 22))
+		if _, err := blockchain.InsertChain(easyBlocks); err != nil {
+			t.Fatalf("failed to insert easy chain: %v", err)
+		}
+		if _, err := blockchain.InsertChain(diffBlocks); err != nil {
+			t.Fatalf("failed to insert difficult chain: %v", err)
+		}
 	} else {
-		bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11), 1)
-		bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22), 1)
+		easyHeaders := make([]*types.Header, len(easyBlocks))
+		for i, block := range easyBlocks {
+			easyHeaders[i] = block.Header()
+		}
+		diffHeaders := make([]*types.Header, len(diffBlocks))
+		for i, block := range diffBlocks {
+			diffHeaders[i] = block.Header()
+		}
+		if _, err := blockchain.InsertHeaderChain(easyHeaders, 1); err != nil {
+			t.Fatalf("failed to insert easy chain: %v", err)
+		}
+		if _, err := blockchain.InsertHeaderChain(diffHeaders, 1); err != nil {
+			t.Fatalf("failed to insert difficult chain: %v", err)
+		}
 	}
 	// Check that the chain is valid number and link wise
 	if full {
-		prev := bc.CurrentBlock()
-		for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
+		prev := blockchain.CurrentBlock()
+		for block := blockchain.GetBlockByNumber(blockchain.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, blockchain.GetBlockByNumber(block.NumberU64()-1) {
 			if prev.ParentHash() != block.Hash() {
 				t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash())
 			}
 		}
 	} else {
-		prev := bc.CurrentHeader()
-		for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) {
+		prev := blockchain.CurrentHeader()
+		for header := blockchain.GetHeaderByNumber(blockchain.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, blockchain.GetHeaderByNumber(header.Number.Uint64()-1) {
 			if prev.ParentHash != header.Hash() {
 				t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash())
 			}
 		}
 	}
 	// Make sure the chain total difficulty is the correct one
-	want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td))
+	want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td))
 	if full {
-		if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 {
+		if have := blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()); have.Cmp(want) != 0 {
 			t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
 		}
 	} else {
-		if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 {
+		if have := blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()); have.Cmp(want) != 0 {
 			t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
 		}
 	}
@@ -443,19 +423,28 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) }
 func TestBadBlockHashes(t *testing.T)  { testBadHashes(t, true) }
 
 func testBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain(true)
-	defer bc.Stop()
+	// Create a pristine chain and database
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
 	// Create a chain, ban a hash and try to import
-	var err error
 	if full {
-		blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10)
+		blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10)
+
 		BadHashes[blocks[2].Header().Hash()] = true
-		_, err = bc.InsertChain(blocks)
+		defer func() { delete(BadHashes, blocks[2].Header().Hash()) }()
+
+		_, err = blockchain.InsertChain(blocks)
 	} else {
-		headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10)
+		headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10)
+
 		BadHashes[headers[2].Hash()] = true
-		_, err = bc.InsertHeaderChain(headers, 1)
+		defer func() { delete(BadHashes, headers[2].Hash()) }()
+
+		_, err = blockchain.InsertHeaderChain(headers, 1)
 	}
 	if err != ErrBlacklistedHash {
 		t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash)
@@ -468,40 +457,41 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) }
 func TestReorgBadBlockHashes(t *testing.T)  { testReorgBadHashes(t, true) }
 
 func testReorgBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain(true)
-	defer bc.Stop()
-
+	// Create a pristine chain and database
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
 	// Create a chain, import and ban afterwards
-	headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10)
-	blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10)
+	headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10)
+	blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10)
 
 	if full {
-		if _, err := bc.InsertChain(blocks); err != nil {
-			t.Fatalf("failed to import blocks: %v", err)
+		if _, err = blockchain.InsertChain(blocks); err != nil {
+			t.Errorf("failed to import blocks: %v", err)
 		}
-		if bc.CurrentBlock().Hash() != blocks[3].Hash() {
-			t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash())
+		if blockchain.CurrentBlock().Hash() != blocks[3].Hash() {
+			t.Errorf("last block hash mismatch: have: %x, want %x", blockchain.CurrentBlock().Hash(), blocks[3].Header().Hash())
 		}
 		BadHashes[blocks[3].Header().Hash()] = true
 		defer func() { delete(BadHashes, blocks[3].Header().Hash()) }()
 	} else {
-		if _, err := bc.InsertHeaderChain(headers, 1); err != nil {
-			t.Fatalf("failed to import headers: %v", err)
+		if _, err = blockchain.InsertHeaderChain(headers, 1); err != nil {
+			t.Errorf("failed to import headers: %v", err)
 		}
-		if bc.CurrentHeader().Hash() != headers[3].Hash() {
-			t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash())
+		if blockchain.CurrentHeader().Hash() != headers[3].Hash() {
+			t.Errorf("last header hash mismatch: have: %x, want %x", blockchain.CurrentHeader().Hash(), headers[3].Hash())
 		}
 		BadHashes[headers[3].Hash()] = true
 		defer func() { delete(BadHashes, headers[3].Hash()) }()
 	}
+	blockchain.Stop()
 
 	// Create a new BlockChain and check that it rolled back the state.
-	ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{})
+	ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{})
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
-	defer ncm.Stop()
-
 	if full {
 		if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() {
 			t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash())
@@ -514,6 +504,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 			t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash())
 		}
 	}
+	ncm.Stop()
 }
 
 // Tests chain insertions in the face of one entity containing an invalid nonce.
@@ -989,10 +980,13 @@ done:
 
 // Tests if the canonical block can be fetched from the database during chain insertion.
 func TestCanonicalBlockRetrieval(t *testing.T) {
-	bc := newTestBlockChain(true)
-	defer bc.Stop()
+	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
-	chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {})
+	chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {})
 
 	var pend sync.WaitGroup
 	pend.Add(len(chain))
@@ -1003,14 +997,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 
 			// try to retrieve a block by its canonical hash and see if the block data can be retrieved.
 			for {
-				ch := GetCanonicalHash(bc.db, block.NumberU64())
+				ch := GetCanonicalHash(blockchain.db, block.NumberU64())
 				if ch == (common.Hash{}) {
 					continue // busy wait for canonical hash to be written
 				}
 				if ch != block.Hash() {
 					t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex())
 				}
-				fb := GetBlock(bc.db, ch, block.NumberU64())
+				fb := GetBlock(blockchain.db, ch, block.NumberU64())
 				if fb == nil {
 					t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex())
 				}
@@ -1021,7 +1015,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 			}
 		}(chain[i])
 
-		if _, err := bc.InsertChain(types.Blocks{chain[i]}); err != nil {
+		if _, err := blockchain.InsertChain(types.Blocks{chain[i]}); err != nil {
 			t.Fatalf("failed to insert block %d: %v", i, err)
 		}
 	}
diff --git a/core/database_util.go b/core/database_util.go
index c6b125dae9..61ab701349 100644
--- a/core/database_util.go
+++ b/core/database_util.go
@@ -70,8 +70,8 @@ var (
 
 	ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error
 
-	preimageCounter    = metrics.NewCounter("db/preimage/total")
-	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
+	preimageCounter    = metrics.NewRegisteredCounter("db/preimage/total", nil)
+	preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
 )
 
 // TxLookupEntry is a positional metadata to help looking up the data content of
diff --git a/core/fees.go b/core/fees.go
deleted file mode 100644
index 83275ea36c..0000000000
--- a/core/fees.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package core
-
-import (
-	"math/big"
-)
-
-var BlockReward = big.NewInt(5e+18)
diff --git a/core/genesis_test.go b/core/genesis_test.go
index cd548d4b1d..052ded6991 100644
--- a/core/genesis_test.go
+++ b/core/genesis_test.go
@@ -118,10 +118,12 @@ func TestSetupGenesis(t *testing.T) {
 				// Commit the 'old' genesis block with Homestead transition at #2.
 				// Advance to block #4, past the homestead transition block of customg.
 				genesis := oldcustomg.MustCommit(db)
+
 				bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
 				defer bc.Stop()
-				bc.SetValidator(bproc{})
-				bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0))
+
+				blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil)
+				bc.InsertChain(blocks)
 				bc.CurrentBlock()
 				// This should return a compatibility error.
 				return SetupGenesisBlock(db, &customg)
diff --git a/core/headerchain.go b/core/headerchain.go
index 0e5215293d..73cd5d2c41 100644
--- a/core/headerchain.go
+++ b/core/headerchain.go
@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/hashicorp/golang-lru"
+	"sync/atomic"
 )
 
 const (
@@ -51,8 +52,8 @@ type HeaderChain struct {
 	chainDb       ethdb.Database
 	genesisHeader *types.Header
 
-	currentHeader     *types.Header // Current head of the header chain (may be above the block chain!)
-	currentHeaderHash common.Hash   // Hash of the current head of the header chain (prevent recomputing all the time)
+	currentHeader     atomic.Value // Current head of the header chain (may be above the block chain!)
+	currentHeaderHash common.Hash  // Hash of the current head of the header chain (prevent recomputing all the time)
 
 	headerCache *lru.Cache // Cache for the most recent block headers
 	tdCache     *lru.Cache // Cache for the most recent block total difficulties
@@ -95,13 +96,13 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
 		return nil, ErrNoGenesis
 	}
 
-	hc.currentHeader = hc.genesisHeader
+	hc.currentHeader.Store(hc.genesisHeader)
 	if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) {
 		if chead := hc.GetHeaderByHash(head); chead != nil {
-			hc.currentHeader = chead
+			hc.currentHeader.Store(chead)
 		}
 	}
-	hc.currentHeaderHash = hc.currentHeader.Hash()
+	hc.currentHeaderHash = hc.CurrentHeader().Hash()
 
 	return hc, nil
 }
@@ -139,7 +140,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 	if ptd == nil {
 		return NonStatTy, consensus.ErrUnknownAncestor
 	}
-	localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64())
+	localTd := hc.GetTd(hc.currentHeaderHash, hc.CurrentHeader().Number.Uint64())
 	externTd := new(big.Int).Add(header.Difficulty, ptd)
 
 	// Irrelevant of the canonical status, write the td and header to the database
@@ -181,7 +182,8 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 		if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil {
 			log.Crit("Failed to insert head header hash", "err", err)
 		}
-		hc.currentHeaderHash, hc.currentHeader = hash, types.CopyHeader(header)
+		hc.currentHeaderHash = hash
+		hc.currentHeader.Store(types.CopyHeader(header))
 
 		status = CanonStatTy
 	} else {
@@ -383,7 +385,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header {
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (hc *HeaderChain) CurrentHeader() *types.Header {
-	return hc.currentHeader
+	return hc.currentHeader.Load().(*types.Header)
 }
 
 // SetCurrentHeader sets the current head header of the canonical chain.
@@ -391,7 +393,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
 	if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil {
 		log.Crit("Failed to insert head header hash", "err", err)
 	}
-	hc.currentHeader = head
+	hc.currentHeader.Store(head)
 	hc.currentHeaderHash = head.Hash()
 }
 
@@ -403,19 +405,20 @@ type DeleteCallback func(common.Hash, uint64)
 // will be deleted and the new one set.
 func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) {
 	height := uint64(0)
-	if hc.currentHeader != nil {
-		height = hc.currentHeader.Number.Uint64()
+
+	if hdr := hc.CurrentHeader(); hdr != nil {
+		height = hdr.Number.Uint64()
 	}
 
-	for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head {
-		hash := hc.currentHeader.Hash()
-		num := hc.currentHeader.Number.Uint64()
+	for hdr := hc.CurrentHeader(); hdr != nil && hdr.Number.Uint64() > head; hdr = hc.CurrentHeader() {
+		hash := hdr.Hash()
+		num := hdr.Number.Uint64()
 		if delFn != nil {
 			delFn(hash, num)
 		}
 		DeleteHeader(hc.chainDb, hash, num)
 		DeleteTd(hc.chainDb, hash, num)
-		hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1)
+		hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1))
 	}
 	// Roll back the canonical chain numbering
 	for i := height; i > head; i-- {
@@ -426,10 +429,10 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) {
 	hc.tdCache.Purge()
 	hc.numberCache.Purge()
 
-	if hc.currentHeader == nil {
-		hc.currentHeader = hc.genesisHeader
+	if hc.CurrentHeader() == nil {
+		hc.currentHeader.Store(hc.genesisHeader)
 	}
-	hc.currentHeaderHash = hc.currentHeader.Hash()
+	hc.currentHeaderHash = hc.CurrentHeader().Hash()
 
 	if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil {
 		log.Crit("Failed to reset head header hash", "err", err)
diff --git a/core/state_transition.go b/core/state_transition.go
index 390473fffd..b19bc12e42 100644
--- a/core/state_transition.go
+++ b/core/state_transition.go
@@ -215,6 +215,9 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 
 	// Pay intrinsic gas
 	gas, err := IntrinsicGas(st.data, contractCreation, homestead)
+	if err != nil {
+		return nil, 0, false, err
+	}
 	if err = st.useGas(gas); err != nil {
 		return nil, 0, false, err
 	}
diff --git a/core/tx_pool.go b/core/tx_pool.go
index dc3ddc4232..0534fe57a1 100644
--- a/core/tx_pool.go
+++ b/core/tx_pool.go
@@ -87,20 +87,20 @@ var (
 
 var (
 	// Metrics for the pending pool
-	pendingDiscardCounter   = metrics.NewCounter("txpool/pending/discard")
-	pendingReplaceCounter   = metrics.NewCounter("txpool/pending/replace")
-	pendingRateLimitCounter = metrics.NewCounter("txpool/pending/ratelimit") // Dropped due to rate limiting
-	pendingNofundsCounter   = metrics.NewCounter("txpool/pending/nofunds")   // Dropped due to out-of-funds
+	pendingDiscardCounter   = metrics.NewRegisteredCounter("txpool/pending/discard", nil)
+	pendingReplaceCounter   = metrics.NewRegisteredCounter("txpool/pending/replace", nil)
+	pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting
+	pendingNofundsCounter   = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil)   // Dropped due to out-of-funds
 
 	// Metrics for the queued pool
-	queuedDiscardCounter   = metrics.NewCounter("txpool/queued/discard")
-	queuedReplaceCounter   = metrics.NewCounter("txpool/queued/replace")
-	queuedRateLimitCounter = metrics.NewCounter("txpool/queued/ratelimit") // Dropped due to rate limiting
-	queuedNofundsCounter   = metrics.NewCounter("txpool/queued/nofunds")   // Dropped due to out-of-funds
+	queuedDiscardCounter   = metrics.NewRegisteredCounter("txpool/queued/discard", nil)
+	queuedReplaceCounter   = metrics.NewRegisteredCounter("txpool/queued/replace", nil)
+	queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting
+	queuedNofundsCounter   = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil)   // Dropped due to out-of-funds
 
 	// General tx metrics
-	invalidTxCounter     = metrics.NewCounter("txpool/invalid")
-	underpricedTxCounter = metrics.NewCounter("txpool/underpriced")
+	invalidTxCounter     = metrics.NewRegisteredCounter("txpool/invalid", nil)
+	underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil)
 )
 
 // TxStatus is the current status of a transaction as seen by the pool.
diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go
index 513651835a..96083337c9 100644
--- a/core/vm/contracts_test.go
+++ b/core/vm/contracts_test.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
 package vm
 
 import (
diff --git a/core/vm/instructions.go b/core/vm/instructions.go
index 7661725010..6daf4e10df 100644
--- a/core/vm/instructions.go
+++ b/core/vm/instructions.go
@@ -302,6 +302,66 @@ func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S
 	return nil, nil
 }
 
+// opSHL implements Shift Left
+// The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the left by arg1 number of bits.
+func opSHL(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards
+	shift, value := math.U256(stack.pop()), math.U256(stack.peek())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		value.SetUint64(0)
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	math.U256(value.Lsh(value, n))
+
+	return nil, nil
+}
+
+// opSHR implements Logical Shift Right
+// The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill.
+func opSHR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards
+	shift, value := math.U256(stack.pop()), math.U256(stack.peek())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		value.SetUint64(0)
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	math.U256(value.Rsh(value, n))
+
+	return nil, nil
+}
+
+// opSAR implements Arithmetic Shift Right
+// The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension.
+func opSAR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, S256 returns (potentially) a new bigint, so we're popping, not peeking this one
+	shift, value := math.U256(stack.pop()), math.S256(stack.pop())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		if value.Sign() > 0 {
+			value.SetUint64(0)
+		} else {
+			value.SetInt64(-1)
+		}
+		stack.push(math.U256(value))
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	value.Rsh(value, n)
+	stack.push(math.U256(value))
+
+	return nil, nil
+}
+
 func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	offset, size := stack.pop(), stack.pop()
 	data := memory.Get(offset.Int64(), size.Int64())
diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go
index 18644989c1..eef4328bde 100644
--- a/core/vm/instructions_test.go
+++ b/core/vm/instructions_test.go
@@ -1,3 +1,19 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
 package vm
 
 import (
@@ -8,6 +24,48 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
+type twoOperandTest struct {
+	x        string
+	y        string
+	expected string
+}
+
+func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) {
+	var (
+		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		stack = newstack()
+		pc    = uint64(0)
+	)
+	for i, test := range tests {
+		x := new(big.Int).SetBytes(common.Hex2Bytes(test.x))
+		shift := new(big.Int).SetBytes(common.Hex2Bytes(test.y))
+		expected := new(big.Int).SetBytes(common.Hex2Bytes(test.expected))
+		stack.push(x)
+		stack.push(shift)
+		opFn(&pc, env, nil, nil, stack)
+		actual := stack.pop()
+		if actual.Cmp(expected) != 0 {
+			t.Errorf("Testcase %d, expected  %v, got %v", i, expected, actual)
+		}
+		// Check pool usage
+		// 1.pool is not allowed to contain anything on the stack
+		// 2.pool is not allowed to contain the same pointers twice
+		if env.interpreter.intPool.pool.len() > 0 {
+
+			poolvals := make(map[*big.Int]struct{})
+			poolvals[actual] = struct{}{}
+
+			for env.interpreter.intPool.pool.len() > 0 {
+				key := env.interpreter.intPool.get()
+				if _, exist := poolvals[key]; exist {
+					t.Errorf("Testcase %d, pool contains double-entry", i)
+				}
+				poolvals[key] = struct{}{}
+			}
+		}
+	}
+}
+
 func TestByteOp(t *testing.T) {
 	var (
 		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
@@ -41,6 +99,98 @@ func TestByteOp(t *testing.T) {
 	}
 }
 
+func TestSHL(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shl-shift-left
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000002"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ff", "8000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0101", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "8000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
+	}
+	testTwoOperandOp(t, tests, opSHL)
+}
+
+func TestSHR(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shr-logical-shift-right
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "01", "4000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "ff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0101", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+	testTwoOperandOp(t, tests, opSHR)
+}
+
+func TestSAR(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#sar-arithmetic-shift-right
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "01", "c000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0101", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"4000000000000000000000000000000000000000000000000000000000000000", "fe", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "f8", "000000000000000000000000000000000000000000000000000000000000007f"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "fe", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+
+	testTwoOperandOp(t, tests, opSAR)
+}
+
+func TestSGT(t *testing.T) {
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+	testTwoOperandOp(t, tests, opSgt)
+}
+
+func TestSLT(t *testing.T) {
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+	}
+	testTwoOperandOp(t, tests, opSlt)
+}
+
 func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) {
 	var (
 		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
@@ -243,3 +393,22 @@ func BenchmarkOpMulmod(b *testing.B) {
 
 	opBenchmark(b, opMulmod, x, y, z)
 }
+
+func BenchmarkOpSHL(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSHL, x, y)
+}
+func BenchmarkOpSHR(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSHR, x, y)
+}
+func BenchmarkOpSAR(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSAR, x, y)
+}
diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go
index 482e67a3a9..e12dd7e5a1 100644
--- a/core/vm/interpreter.go
+++ b/core/vm/interpreter.go
@@ -20,9 +20,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -70,6 +68,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter {
 	// we'll set the default jump table.
 	if !cfg.JumpTable[STOP].valid {
 		switch {
+		case evm.ChainConfig().IsConstantinople(evm.BlockNumber):
+			cfg.JumpTable = constantinopleInstructionSet
 		case evm.ChainConfig().IsByzantium(evm.BlockNumber):
 			cfg.JumpTable = byzantiumInstructionSet
 		case evm.ChainConfig().IsHomestead(evm.BlockNumber):
@@ -123,11 +123,6 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 		return nil, nil
 	}
 
-	codehash := contract.CodeHash // codehash is used when doing jump dest caching
-	if codehash == (common.Hash{}) {
-		codehash = crypto.Keccak256Hash(contract.Code)
-	}
-
 	var (
 		op    OpCode        // current opcode
 		mem   = NewMemory() // bound memory
diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go
index a1c5ad9c67..3389941353 100644
--- a/core/vm/jump_table.go
+++ b/core/vm/jump_table.go
@@ -51,11 +51,38 @@ type operation struct {
 }
 
 var (
-	frontierInstructionSet  = NewFrontierInstructionSet()
-	homesteadInstructionSet = NewHomesteadInstructionSet()
-	byzantiumInstructionSet = NewByzantiumInstructionSet()
+	frontierInstructionSet       = NewFrontierInstructionSet()
+	homesteadInstructionSet      = NewHomesteadInstructionSet()
+	byzantiumInstructionSet      = NewByzantiumInstructionSet()
+	constantinopleInstructionSet = NewConstantinopleInstructionSet()
 )
 
+// NewConstantinopleInstructionSet returns the frontier, homestead
+// byzantium and contantinople instructions.
+func NewConstantinopleInstructionSet() [256]operation {
+	// instructions that can be executed during the byzantium phase.
+	instructionSet := NewByzantiumInstructionSet()
+	instructionSet[SHL] = operation{
+		execute:       opSHL,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	instructionSet[SHR] = operation{
+		execute:       opSHR,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	instructionSet[SAR] = operation{
+		execute:       opSAR,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	return instructionSet
+}
+
 // NewByzantiumInstructionSet returns the frontier, homestead and
 // byzantium instructions.
 func NewByzantiumInstructionSet() [256]operation {
diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go
index 0c65507355..7fe55b72f6 100644
--- a/core/vm/opcodes.go
+++ b/core/vm/opcodes.go
@@ -63,6 +63,9 @@ const (
 	XOR
 	NOT
 	BYTE
+	SHL
+	SHR
+	SAR
 
 	SHA3 = 0x20
 )
@@ -234,6 +237,9 @@ var opCodeToString = map[OpCode]string{
 	OR:     "OR",
 	XOR:    "XOR",
 	BYTE:   "BYTE",
+	SHL:    "SHL",
+	SHR:    "SHR",
+	SAR:    "SAR",
 	ADDMOD: "ADDMOD",
 	MULMOD: "MULMOD",
 
@@ -400,6 +406,9 @@ var stringToOp = map[string]OpCode{
 	"OR":             OR,
 	"XOR":            XOR,
 	"BYTE":           BYTE,
+	"SHL":            SHL,
+	"SHR":            SHR,
+	"SAR":            SAR,
 	"ADDMOD":         ADDMOD,
 	"MULMOD":         MULMOD,
 	"SHA3":           SHA3,
diff --git a/dashboard/dashboard.go b/dashboard/dashboard.go
index 09038638ed..2ca795187f 100644
--- a/dashboard/dashboard.go
+++ b/dashboard/dashboard.go
@@ -36,10 +36,10 @@ import (
 
 	"github.com/elastic/gosigar"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/rcrowley/go-metrics"
 	"golang.org/x/net/websocket"
 )
 
diff --git a/eth/api_backend.go b/eth/api_backend.go
index 91f392f94f..ecd5488a24 100644
--- a/eth/api_backend.go
+++ b/eth/api_backend.go
@@ -104,6 +104,18 @@ func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash)
 	return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil
 }
 
+func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
+	if receipts == nil {
+		return nil, nil
+	}
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
+}
+
 func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int {
 	return b.eth.blockchain.GetTdByHash(blockHash)
 }
diff --git a/eth/api_test.go b/eth/api_test.go
index 248bc3ab6a..900a82bb6a 100644
--- a/eth/api_test.go
+++ b/eth/api_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index 7f490d9e9b..d13247766a 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -31,8 +31,8 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/rcrowley/go-metrics"
 )
 
 var (
@@ -173,8 +173,8 @@ type LightChain interface {
 type BlockChain interface {
 	LightChain
 
-	// HasBlockAndState verifies block and associated states' presence in the local chain.
-	HasBlockAndState(common.Hash, uint64) bool
+	// HasBlock verifies a block's presence in the local chain.
+	HasBlock(common.Hash, uint64) bool
 
 	// GetBlockByHash retrieves a block from the local chain.
 	GetBlockByHash(common.Hash) *types.Block
@@ -266,7 +266,6 @@ func (d *Downloader) Synchronising() bool {
 // RegisterPeer injects a new download peer into the set of block source to be
 // used for fetching hashes and blocks from.
 func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error {
-
 	logger := log.New("peer", id)
 	logger.Trace("Registering sync peer")
 	if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil {
@@ -583,7 +582,6 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 	// Figure out the valid ancestor range to prevent rewrite attacks
 	floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
 
-	p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
 	if d.mode == FullSync {
 		ceil = d.blockchain.CurrentBlock().NumberU64()
 	} else if d.mode == FastSync {
@@ -592,6 +590,8 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 	if ceil >= MaxForkAncestry {
 		floor = int64(ceil - MaxForkAncestry)
 	}
+	p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
+
 	// Request the topmost blocks to short circuit binary ancestor lookup
 	head := ceil
 	if head > height {
@@ -647,7 +647,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					continue
 				}
 				// Otherwise check if we already know the header or not
-				if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
+				if (d.mode == FullSync && d.blockchain.HasBlock(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
 					number, hash = headers[i].Number.Uint64(), headers[i].Hash()
 
 					// If every header is known, even future ones, the peer straight out lied about its head
@@ -712,7 +712,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				arrived = true
 
 				// Modify the search interval based on the response
-				if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
+				if (d.mode == FullSync && !d.blockchain.HasBlock(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
 					end = check
 					break
 				}
diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go
index d94d55f114..cb671a7df4 100644
--- a/eth/downloader/downloader_test.go
+++ b/eth/downloader/downloader_test.go
@@ -221,14 +221,9 @@ func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool {
 	return dl.GetHeaderByHash(hash) != nil
 }
 
-// HasBlockAndState checks if a block and associated state is present in the testers canonical chain.
-func (dl *downloadTester) HasBlockAndState(hash common.Hash, number uint64) bool {
-	block := dl.GetBlockByHash(hash)
-	if block == nil {
-		return false
-	}
-	_, err := dl.stateDb.Get(block.Root().Bytes())
-	return err == nil
+// HasBlock checks if a block is present in the testers canonical chain.
+func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool {
+	return dl.GetBlockByHash(hash) != nil
 }
 
 // GetHeader retrieves a header from the testers canonical chain.
diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go
index 58764ccf06..d4eb337946 100644
--- a/eth/downloader/metrics.go
+++ b/eth/downloader/metrics.go
@@ -23,21 +23,21 @@ import (
 )
 
 var (
-	headerInMeter      = metrics.NewMeter("eth/downloader/headers/in")
-	headerReqTimer     = metrics.NewTimer("eth/downloader/headers/req")
-	headerDropMeter    = metrics.NewMeter("eth/downloader/headers/drop")
-	headerTimeoutMeter = metrics.NewMeter("eth/downloader/headers/timeout")
+	headerInMeter      = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil)
+	headerReqTimer     = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil)
+	headerDropMeter    = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil)
+	headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil)
 
-	bodyInMeter      = metrics.NewMeter("eth/downloader/bodies/in")
-	bodyReqTimer     = metrics.NewTimer("eth/downloader/bodies/req")
-	bodyDropMeter    = metrics.NewMeter("eth/downloader/bodies/drop")
-	bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout")
+	bodyInMeter      = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil)
+	bodyReqTimer     = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil)
+	bodyDropMeter    = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil)
+	bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil)
 
-	receiptInMeter      = metrics.NewMeter("eth/downloader/receipts/in")
-	receiptReqTimer     = metrics.NewTimer("eth/downloader/receipts/req")
-	receiptDropMeter    = metrics.NewMeter("eth/downloader/receipts/drop")
-	receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout")
+	receiptInMeter      = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil)
+	receiptReqTimer     = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil)
+	receiptDropMeter    = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil)
+	receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil)
 
-	stateInMeter   = metrics.NewMeter("eth/downloader/states/in")
-	stateDropMeter = metrics.NewMeter("eth/downloader/states/drop")
+	stateInMeter   = metrics.NewRegisteredMeter("eth/downloader/states/in", nil)
+	stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil)
 )
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index a1a70e46ea..359cce54b5 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -28,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/rcrowley/go-metrics"
+	"github.com/ethereum/go-ethereum/metrics"
 	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
 )
 
diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go
index 1ed8075bf3..d68d12f000 100644
--- a/eth/fetcher/metrics.go
+++ b/eth/fetcher/metrics.go
@@ -23,21 +23,21 @@ import (
 )
 
 var (
-	propAnnounceInMeter   = metrics.NewMeter("eth/fetcher/prop/announces/in")
-	propAnnounceOutTimer  = metrics.NewTimer("eth/fetcher/prop/announces/out")
-	propAnnounceDropMeter = metrics.NewMeter("eth/fetcher/prop/announces/drop")
-	propAnnounceDOSMeter  = metrics.NewMeter("eth/fetcher/prop/announces/dos")
+	propAnnounceInMeter   = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil)
+	propAnnounceOutTimer  = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil)
+	propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil)
+	propAnnounceDOSMeter  = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil)
 
-	propBroadcastInMeter   = metrics.NewMeter("eth/fetcher/prop/broadcasts/in")
-	propBroadcastOutTimer  = metrics.NewTimer("eth/fetcher/prop/broadcasts/out")
-	propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop")
-	propBroadcastDOSMeter  = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos")
+	propBroadcastInMeter   = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil)
+	propBroadcastOutTimer  = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil)
+	propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil)
+	propBroadcastDOSMeter  = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil)
 
-	headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers")
-	bodyFetchMeter   = metrics.NewMeter("eth/fetcher/fetch/bodies")
+	headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil)
+	bodyFetchMeter   = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil)
 
-	headerFilterInMeter  = metrics.NewMeter("eth/fetcher/filter/headers/in")
-	headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out")
-	bodyFilterInMeter    = metrics.NewMeter("eth/fetcher/filter/bodies/in")
-	bodyFilterOutMeter   = metrics.NewMeter("eth/fetcher/filter/bodies/out")
+	headerFilterInMeter  = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil)
+	headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil)
+	bodyFilterInMeter    = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil)
+	bodyFilterOutMeter   = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil)
 )
diff --git a/eth/filters/filter.go b/eth/filters/filter.go
index 43d7e2a812..5dfe60e772 100644
--- a/eth/filters/filter.go
+++ b/eth/filters/filter.go
@@ -34,6 +34,7 @@ type Backend interface {
 	EventMux() *event.TypeMux
 	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
 	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
 
 	SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription
 	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
@@ -201,16 +202,28 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e
 // match the filter criteria. This function is called when the bloom filter signals a potential match.
 func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) {
 	// Get the logs of the block
-	receipts, err := f.backend.GetReceipts(ctx, header.Hash())
+	logsList, err := f.backend.GetLogs(ctx, header.Hash())
 	if err != nil {
 		return nil, err
 	}
 	var unfiltered []*types.Log
-	for _, receipt := range receipts {
-		unfiltered = append(unfiltered, receipt.Logs...)
+	for _, logs := range logsList {
+		unfiltered = append(unfiltered, logs...)
 	}
 	logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
 	if len(logs) > 0 {
+		// We have matching logs, check if we need to resolve full logs via the light client
+		if logs[0].TxHash == (common.Hash{}) {
+			receipts, err := f.backend.GetReceipts(ctx, header.Hash())
+			if err != nil {
+				return nil, err
+			}
+			unfiltered = unfiltered[:0]
+			for _, receipt := range receipts {
+				unfiltered = append(unfiltered, receipt.Logs...)
+			}
+			logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
+		}
 		return logs, nil
 	}
 	return nil, nil
diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go
index b09998f9c7..f8097c7b95 100644
--- a/eth/filters/filter_system.go
+++ b/eth/filters/filter_system.go
@@ -375,19 +375,35 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.
 		// Get the logs of the block
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancel()
-		receipts, err := es.backend.GetReceipts(ctx, header.Hash())
+		logsList, err := es.backend.GetLogs(ctx, header.Hash())
 		if err != nil {
 			return nil
 		}
 		var unfiltered []*types.Log
-		for _, receipt := range receipts {
-			for _, log := range receipt.Logs {
+		for _, logs := range logsList {
+			for _, log := range logs {
 				logcopy := *log
 				logcopy.Removed = remove
 				unfiltered = append(unfiltered, &logcopy)
 			}
 		}
 		logs := filterLogs(unfiltered, nil, nil, addresses, topics)
+		if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) {
+			// We have matching but non-derived logs
+			receipts, err := es.backend.GetReceipts(ctx, header.Hash())
+			if err != nil {
+				return nil
+			}
+			unfiltered = unfiltered[:0]
+			for _, receipt := range receipts {
+				for _, log := range receipt.Logs {
+					logcopy := *log
+					logcopy.Removed = remove
+					unfiltered = append(unfiltered, &logcopy)
+				}
+			}
+			logs = filterLogs(unfiltered, nil, nil, addresses, topics)
+		}
 		return logs
 	}
 	return nil
diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go
index 7ec3b4be7f..61761151a2 100644
--- a/eth/filters/filter_system_test.go
+++ b/eth/filters/filter_system_test.go
@@ -69,8 +69,19 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumbe
 }
 
 func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) {
-	num := core.GetBlockNumber(b.db, blockHash)
-	return core.GetBlockReceipts(b.db, blockHash, num), nil
+	number := core.GetBlockNumber(b.db, blockHash)
+	return core.GetBlockReceipts(b.db, blockHash, number), nil
+}
+
+func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	number := core.GetBlockNumber(b.db, blockHash)
+	receipts := core.GetBlockReceipts(b.db, blockHash, number)
+
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
 }
 
 func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription {
diff --git a/eth/handler.go b/eth/handler.go
index c2426544f6..3fae0cd00d 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -249,7 +249,8 @@ func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *p
 // handle is the callback invoked to manage the life cycle of an eth peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 	p.Log().Debug("Ethereum peer connected", "name", p.Name())
diff --git a/eth/metrics.go b/eth/metrics.go
index 5fa2597d4d..0533a2a875 100644
--- a/eth/metrics.go
+++ b/eth/metrics.go
@@ -22,38 +22,38 @@ import (
 )
 
 var (
-	propTxnInPacketsMeter     = metrics.NewMeter("eth/prop/txns/in/packets")
-	propTxnInTrafficMeter     = metrics.NewMeter("eth/prop/txns/in/traffic")
-	propTxnOutPacketsMeter    = metrics.NewMeter("eth/prop/txns/out/packets")
-	propTxnOutTrafficMeter    = metrics.NewMeter("eth/prop/txns/out/traffic")
-	propHashInPacketsMeter    = metrics.NewMeter("eth/prop/hashes/in/packets")
-	propHashInTrafficMeter    = metrics.NewMeter("eth/prop/hashes/in/traffic")
-	propHashOutPacketsMeter   = metrics.NewMeter("eth/prop/hashes/out/packets")
-	propHashOutTrafficMeter   = metrics.NewMeter("eth/prop/hashes/out/traffic")
-	propBlockInPacketsMeter   = metrics.NewMeter("eth/prop/blocks/in/packets")
-	propBlockInTrafficMeter   = metrics.NewMeter("eth/prop/blocks/in/traffic")
-	propBlockOutPacketsMeter  = metrics.NewMeter("eth/prop/blocks/out/packets")
-	propBlockOutTrafficMeter  = metrics.NewMeter("eth/prop/blocks/out/traffic")
-	reqHeaderInPacketsMeter   = metrics.NewMeter("eth/req/headers/in/packets")
-	reqHeaderInTrafficMeter   = metrics.NewMeter("eth/req/headers/in/traffic")
-	reqHeaderOutPacketsMeter  = metrics.NewMeter("eth/req/headers/out/packets")
-	reqHeaderOutTrafficMeter  = metrics.NewMeter("eth/req/headers/out/traffic")
-	reqBodyInPacketsMeter     = metrics.NewMeter("eth/req/bodies/in/packets")
-	reqBodyInTrafficMeter     = metrics.NewMeter("eth/req/bodies/in/traffic")
-	reqBodyOutPacketsMeter    = metrics.NewMeter("eth/req/bodies/out/packets")
-	reqBodyOutTrafficMeter    = metrics.NewMeter("eth/req/bodies/out/traffic")
-	reqStateInPacketsMeter    = metrics.NewMeter("eth/req/states/in/packets")
-	reqStateInTrafficMeter    = metrics.NewMeter("eth/req/states/in/traffic")
-	reqStateOutPacketsMeter   = metrics.NewMeter("eth/req/states/out/packets")
-	reqStateOutTrafficMeter   = metrics.NewMeter("eth/req/states/out/traffic")
-	reqReceiptInPacketsMeter  = metrics.NewMeter("eth/req/receipts/in/packets")
-	reqReceiptInTrafficMeter  = metrics.NewMeter("eth/req/receipts/in/traffic")
-	reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets")
-	reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")
-	miscInPacketsMeter        = metrics.NewMeter("eth/misc/in/packets")
-	miscInTrafficMeter        = metrics.NewMeter("eth/misc/in/traffic")
-	miscOutPacketsMeter       = metrics.NewMeter("eth/misc/out/packets")
-	miscOutTrafficMeter       = metrics.NewMeter("eth/misc/out/traffic")
+	propTxnInPacketsMeter     = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil)
+	propTxnInTrafficMeter     = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil)
+	propTxnOutPacketsMeter    = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil)
+	propTxnOutTrafficMeter    = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil)
+	propHashInPacketsMeter    = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil)
+	propHashInTrafficMeter    = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil)
+	propHashOutPacketsMeter   = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil)
+	propHashOutTrafficMeter   = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil)
+	propBlockInPacketsMeter   = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil)
+	propBlockInTrafficMeter   = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil)
+	propBlockOutPacketsMeter  = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil)
+	propBlockOutTrafficMeter  = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil)
+	reqHeaderInPacketsMeter   = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil)
+	reqHeaderInTrafficMeter   = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil)
+	reqHeaderOutPacketsMeter  = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil)
+	reqHeaderOutTrafficMeter  = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil)
+	reqBodyInPacketsMeter     = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil)
+	reqBodyInTrafficMeter     = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil)
+	reqBodyOutPacketsMeter    = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil)
+	reqBodyOutTrafficMeter    = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil)
+	reqStateInPacketsMeter    = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil)
+	reqStateInTrafficMeter    = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil)
+	reqStateOutPacketsMeter   = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil)
+	reqStateOutTrafficMeter   = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil)
+	reqReceiptInPacketsMeter  = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil)
+	reqReceiptInTrafficMeter  = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil)
+	reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil)
+	reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil)
+	miscInPacketsMeter        = metrics.NewRegisteredMeter("eth/misc/in/packets", nil)
+	miscInTrafficMeter        = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil)
+	miscOutPacketsMeter       = metrics.NewRegisteredMeter("eth/misc/out/packets", nil)
+	miscOutTrafficMeter       = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil)
 )
 
 // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of
diff --git a/eth/sync.go b/eth/sync.go
index a8ae646170..2da1464bc5 100644
--- a/eth/sync.go
+++ b/eth/sync.go
@@ -189,18 +189,13 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
 		mode = downloader.FastSync
 	}
 	// Run the sync cycle, and disable fast sync if we've went past the pivot block
-	err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode)
-
-	if atomic.LoadUint32(&pm.fastSync) == 1 {
-		// Disable fast sync if we indeed have something in our chain
-		if pm.blockchain.CurrentBlock().NumberU64() > 0 {
-			log.Info("Fast sync complete, auto disabling")
-			atomic.StoreUint32(&pm.fastSync, 0)
-		}
-	}
-	if err != nil {
+	if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil {
 		return
 	}
+	if atomic.LoadUint32(&pm.fastSync) == 1 {
+		log.Info("Fast sync complete, auto disabling")
+		atomic.StoreUint32(&pm.fastSync, 0)
+	}
 	atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done
 	if head := pm.blockchain.CurrentBlock(); head.NumberU64() > 0 {
 		// We've completed a sync cycle, notify all peers of new state. This path is
diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go
index f3f848fc1b..4cec9e633c 100644
--- a/eth/tracers/tracer.go
+++ b/eth/tracers/tracer.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go
index 7224a1489f..117c376b81 100644
--- a/eth/tracers/tracer_test.go
+++ b/eth/tracers/tracer_test.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/ethdb/database.go b/ethdb/database.go
index d86585f079..57d38f7f5f 100644
--- a/ethdb/database.go
+++ b/ethdb/database.go
@@ -29,8 +29,6 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-
-	gometrics "github.com/rcrowley/go-metrics"
 )
 
 var OpenFileLimit = 64
@@ -39,15 +37,15 @@ type LDBDatabase struct {
 	fn string      // filename for reporting
 	db *leveldb.DB // LevelDB instance
 
-	getTimer       gometrics.Timer // Timer for measuring the database get request counts and latencies
-	putTimer       gometrics.Timer // Timer for measuring the database put request counts and latencies
-	delTimer       gometrics.Timer // Timer for measuring the database delete request counts and latencies
-	missMeter      gometrics.Meter // Meter for measuring the missed database get requests
-	readMeter      gometrics.Meter // Meter for measuring the database get request data usage
-	writeMeter     gometrics.Meter // Meter for measuring the database put request data usage
-	compTimeMeter  gometrics.Meter // Meter for measuring the total time spent in database compaction
-	compReadMeter  gometrics.Meter // Meter for measuring the data read during compaction
-	compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction
+	getTimer       metrics.Timer // Timer for measuring the database get request counts and latencies
+	putTimer       metrics.Timer // Timer for measuring the database put request counts and latencies
+	delTimer       metrics.Timer // Timer for measuring the database delete request counts and latencies
+	missMeter      metrics.Meter // Meter for measuring the missed database get requests
+	readMeter      metrics.Meter // Meter for measuring the database get request data usage
+	writeMeter     metrics.Meter // Meter for measuring the database put request data usage
+	compTimeMeter  metrics.Meter // Meter for measuring the total time spent in database compaction
+	compReadMeter  metrics.Meter // Meter for measuring the data read during compaction
+	compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
 
 	quitLock sync.Mutex      // Mutex protecting the quit channel access
 	quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
@@ -180,15 +178,15 @@ func (db *LDBDatabase) Meter(prefix string) {
 		return
 	}
 	// Initialize all the metrics collector at the requested prefix
-	db.getTimer = metrics.NewTimer(prefix + "user/gets")
-	db.putTimer = metrics.NewTimer(prefix + "user/puts")
-	db.delTimer = metrics.NewTimer(prefix + "user/dels")
-	db.missMeter = metrics.NewMeter(prefix + "user/misses")
-	db.readMeter = metrics.NewMeter(prefix + "user/reads")
-	db.writeMeter = metrics.NewMeter(prefix + "user/writes")
-	db.compTimeMeter = metrics.NewMeter(prefix + "compact/time")
-	db.compReadMeter = metrics.NewMeter(prefix + "compact/input")
-	db.compWriteMeter = metrics.NewMeter(prefix + "compact/output")
+	db.getTimer = metrics.NewRegisteredTimer(prefix+"user/gets", nil)
+	db.putTimer = metrics.NewRegisteredTimer(prefix+"user/puts", nil)
+	db.delTimer = metrics.NewRegisteredTimer(prefix+"user/dels", nil)
+	db.missMeter = metrics.NewRegisteredMeter(prefix+"user/misses", nil)
+	db.readMeter = metrics.NewRegisteredMeter(prefix+"user/reads", nil)
+	db.writeMeter = metrics.NewRegisteredMeter(prefix+"user/writes", nil)
+	db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
+	db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
+	db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
 
 	// Create a quit channel for the periodic collector and run it
 	db.quitLock.Lock()
diff --git a/internal/build/env.go b/internal/build/env.go
index c9848bf82c..b553e0ed80 100644
--- a/internal/build/env.go
+++ b/internal/build/env.go
@@ -94,7 +94,7 @@ func LocalEnv() Environment {
 	}
 	if env.Branch == "" {
 		if head != "HEAD" {
-			env.Branch = strings.TrimLeft(head, "refs/heads/")
+			env.Branch = strings.TrimPrefix(head, "refs/heads/")
 		}
 	}
 	if info, err := os.Stat(".git/objects"); err == nil && info.IsDir() && env.Tag == "" {
diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go
index fae61cfe32..20e82ec2a9 100644
--- a/internal/cmdtest/test_cmd.go
+++ b/internal/cmdtest/test_cmd.go
@@ -1,18 +1,18 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see .
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
 
 package cmdtest
 
diff --git a/internal/debug/flags.go b/internal/debug/flags.go
index 6247cc7dc0..1f181bf8b0 100644
--- a/internal/debug/flags.go
+++ b/internal/debug/flags.go
@@ -26,6 +26,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/log/term"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/metrics/exp"
 	colorable "github.com/mattn/go-colorable"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -127,6 +129,10 @@ func Setup(ctx *cli.Context) error {
 
 	// pprof server
 	if ctx.GlobalBool(pprofFlag.Name) {
+		// Hook go-metrics into expvar on any /debug/metrics request, load all vars
+		// from the registry into expvar, and execute regular expvar handler.
+		exp.Exp(metrics.DefaultRegistry)
+
 		address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name))
 		go func() {
 			log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address))
diff --git a/internal/ethapi/addrlock.go b/internal/ethapi/addrlock.go
index 5a9c948b83..61ddff688c 100644
--- a/internal/ethapi/addrlock.go
+++ b/internal/ethapi/addrlock.go
@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go
index 314086335c..20a060e727 100644
--- a/internal/ethapi/api.go
+++ b/internal/ethapi/api.go
@@ -1032,15 +1032,19 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 }
 
 // GetTransactionReceipt returns the transaction receipt for the given transaction hash.
-func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) {
+func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
-		return nil, errors.New("unknown transaction")
+		return nil, nil
 	}
-	receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available
-	if receipt == nil {
-		return nil, errors.New("unknown receipt")
+	receipts, err := s.b.GetReceipts(ctx, blockHash)
+	if err != nil {
+		return nil, err
 	}
+	if len(receipts) <= int(index) {
+		return nil, nil
+	}
+	receipt := receipts[index]
 
 	var signer types.Signer = types.FrontierSigner{}
 	if tx.Protected() {
@@ -1135,6 +1139,18 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error {
 	if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) {
 		return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`)
 	}
+	if args.To == nil {
+		// Contract creation
+		var input []byte
+		if args.Data != nil {
+			input = *args.Data
+		} else if args.Input != nil {
+			input = *args.Input
+		}
+		if len(input) == 0 {
+			return errors.New(`contract creation without any data provided`)
+		}
+	}
 	return nil
 }
 
diff --git a/les/api_backend.go b/les/api_backend.go
index 56f617a7db..3fc5c33a44 100644
--- a/les/api_backend.go
+++ b/les/api_backend.go
@@ -87,6 +87,10 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash)
 	return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
 }
 
+func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
+}
+
 func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int {
 	return b.eth.blockchain.GetTdByHash(blockHash)
 }
diff --git a/les/fetcher.go b/les/fetcher.go
index 3fc4df30b9..e12a2c78a2 100644
--- a/les/fetcher.go
+++ b/les/fetcher.go
@@ -36,24 +36,26 @@ const (
 	maxNodeCount      = 20               // maximum number of fetcherTreeNode entries remembered for each peer
 )
 
-// lightFetcher
+// lightFetcher implements retrieval of newly announced headers. It also provides a peerHasBlock function for the
+// ODR system to ensure that we only request data related to a certain block from peers who have already processed
+// and announced that block.
 type lightFetcher struct {
 	pm    *ProtocolManager
 	odr   *LesOdr
 	chain *light.LightChain
 
+	lock            sync.Mutex // lock protects access to the fetcher's internal state variables except sent requests
 	maxConfirmedTd  *big.Int
 	peers           map[*peer]*fetcherPeerInfo
 	lastUpdateStats *updateStatsEntry
+	syncing         bool
+	syncDone        chan *peer
 
-	lock       sync.Mutex // qwerqwerqwe
-	deliverChn chan fetchResponse
-	reqMu      sync.RWMutex
+	reqMu      sync.RWMutex // reqMu protects access to sent header fetch requests
 	requested  map[uint64]fetchRequest
+	deliverChn chan fetchResponse
 	timeoutChn chan uint64
 	requestChn chan bool // true if initiated from outside
-	syncing    bool
-	syncDone   chan *peer
 }
 
 // fetcherPeerInfo holds fetcher-specific information about each active peer
@@ -425,6 +427,9 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 			},
 			canSend: func(dp distPeer) bool {
 				p := dp.(*peer)
+				f.lock.Lock()
+				defer f.lock.Unlock()
+
 				fp := f.peers[p]
 				return fp != nil && fp.nodeByHash[bestHash] != nil
 			},
@@ -557,8 +562,13 @@ func (f *lightFetcher) checkAnnouncedHeaders(fp *fetcherPeerInfo, headers []*typ
 				return true
 			}
 			// we ran out of recently delivered headers but have not reached a node known by this peer yet, continue matching
-			td = f.chain.GetTd(header.ParentHash, header.Number.Uint64()-1)
-			header = f.chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
+			hash, number := header.ParentHash, header.Number.Uint64()-1
+			td = f.chain.GetTd(hash, number)
+			header = f.chain.GetHeader(hash, number)
+			if header == nil || td == nil {
+				log.Error("Missing parent of validated header", "hash", hash, "number", number)
+				return false
+			}
 		} else {
 			header = headers[i]
 			td = tds[i]
@@ -642,13 +652,18 @@ func (f *lightFetcher) checkKnownNode(p *peer, n *fetcherTreeNode) bool {
 	if td == nil {
 		return false
 	}
+	header := f.chain.GetHeader(n.hash, n.number)
+	// check the availability of both header and td because reads are not protected by chain db mutex
+	// Note: returning false is always safe here
+	if header == nil {
+		return false
+	}
 
 	fp := f.peers[p]
 	if fp == nil {
 		p.Log().Debug("Unknown peer to check known nodes")
 		return false
 	}
-	header := f.chain.GetHeader(n.hash, n.number)
 	if !f.checkAnnouncedHeaders(fp, []*types.Header{header}, []*big.Int{td}) {
 		p.Log().Debug("Inconsistent announcement")
 		go f.pm.removePeer(p.id)
diff --git a/les/handler.go b/les/handler.go
index 5c93133fb7..9627f392be 100644
--- a/les/handler.go
+++ b/les/handler.go
@@ -260,7 +260,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea
 // handle is the callback invoked to manage the life cycle of a les peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 
@@ -790,10 +791,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 				break
 			}
 		}
-		proofs := nodes.NodeList()
 		bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
 		pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
-		return p.SendProofsV2(req.ReqID, bv, proofs)
+		return p.SendProofsV2(req.ReqID, bv, nodes.NodeList())
 
 	case ProofsV1Msg:
 		if pm.odr == nil {
@@ -856,15 +856,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 		if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) {
 			return errResp(ErrRequestRejected, "")
 		}
+		trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix))
 		for _, req := range req.Reqs {
 			if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
-				sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1)
+				sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1)
 				if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) {
-					statedb, err := pm.blockchain.State()
-					if err != nil {
-						continue
-					}
-					trie, err := statedb.Database().OpenTrie(root)
+					trie, err := trie.New(root, trieDb)
 					if err != nil {
 						continue
 					}
@@ -878,7 +875,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 					if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit {
 						break
 					}
-
 				}
 			}
 		}
@@ -910,20 +906,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			lastIdx  uint64
 			lastType uint
 			root     common.Hash
-			statedb  *state.StateDB
-			trie     state.Trie
+			auxTrie  *trie.Trie
 		)
-
 		nodes := light.NewNodeSet()
-
 		for _, req := range req.Reqs {
-			if trie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx {
-				statedb, trie, lastType, lastIdx = nil, nil, req.HelperTrieType, req.TrieIdx
+			if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx {
+				auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx
 
-				if root, _ = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) {
-					if statedb, _ = pm.blockchain.State(); statedb != nil {
-						trie, _ = statedb.Database().OpenTrie(root)
-					}
+				var prefix string
+				if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) {
+					auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix)))
 				}
 			}
 			if req.AuxReq == auxRoot {
@@ -934,8 +926,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 				auxData = append(auxData, data)
 				auxBytes += len(data)
 			} else {
-				if trie != nil {
-					trie.Prove(req.Key, req.FromLevel, nodes)
+				if auxTrie != nil {
+					auxTrie.Prove(req.Key, req.FromLevel, nodes)
 				}
 				if req.AuxReq != 0 {
 					data := pm.getHelperTrieAuxData(req)
@@ -947,10 +939,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 				break
 			}
 		}
-		proofs := nodes.NodeList()
 		bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
 		pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
-		return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: proofs, AuxData: auxData})
+		return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData})
 
 	case HeaderProofsMsg:
 		if pm.odr == nil {
@@ -1123,7 +1114,7 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common.
 func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) {
 	switch id {
 	case htCanonical:
-		sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.ChtFrequency-1)
+		sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1)
 		return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix
 	case htBloomBits:
 		sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1)
@@ -1134,10 +1125,8 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri
 
 // getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request
 func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte {
-	if req.HelperTrieType == htCanonical && req.AuxReq == auxHeader {
-		if len(req.Key) != 8 {
-			return nil
-		}
+	switch {
+	case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8:
 		blockNum := binary.BigEndian.Uint64(req.Key)
 		hash := core.GetCanonicalHash(pm.chainDb, blockNum)
 		return core.GetHeaderRLP(pm.chainDb, hash, blockNum)
diff --git a/les/handler_test.go b/les/handler_test.go
index e5446c031d..9468032f67 100644
--- a/les/handler_test.go
+++ b/les/handler_test.go
@@ -17,7 +17,7 @@
 package les
 
 import (
-	"bytes"
+	"encoding/binary"
 	"math/big"
 	"math/rand"
 	"testing"
@@ -45,27 +45,8 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{}
 	return p2p.ExpectMsg(r, msgcode, resp{reqID, bv, data})
 }
 
-func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) {
-	if exp.KeyCount() > len(got) {
-		t.Errorf("proof has fewer nodes than expected")
-		return
-	}
-	if exp.KeyCount() < len(got) {
-		t.Errorf("proof has more nodes than expected")
-		return
-	}
-	for _, node := range got {
-		n, _ := exp.Get(crypto.Keccak256(node))
-		if !bytes.Equal(n, node) {
-			t.Errorf("proof contents mismatch")
-			return
-		}
-	}
-}
-
 // Tests that block headers can be retrieved from a remote chain based on user queries.
 func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) }
-
 func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) }
 
 func testGetBlockHeaders(t *testing.T, protocol int) {
@@ -196,7 +177,6 @@ func testGetBlockHeaders(t *testing.T, protocol int) {
 
 // Tests that block contents can be retrieved from a remote chain based on their hashes.
 func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) }
-
 func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) }
 
 func testGetBlockBodies(t *testing.T, protocol int) {
@@ -274,7 +254,6 @@ func testGetBlockBodies(t *testing.T, protocol int) {
 
 // Tests that the contract codes can be retrieved based on account addresses.
 func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) }
-
 func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) }
 
 func testGetCode(t *testing.T, protocol int) {
@@ -309,7 +288,6 @@ func testGetCode(t *testing.T, protocol int) {
 
 // Tests that the transaction receipts can be retrieved based on hashes.
 func TestGetReceiptLes1(t *testing.T) { testGetReceipt(t, 1) }
-
 func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) }
 
 func testGetReceipt(t *testing.T, protocol int) {
@@ -338,7 +316,6 @@ func testGetReceipt(t *testing.T, protocol int) {
 
 // Tests that trie merkle proofs can be retrieved
 func TestGetProofsLes1(t *testing.T) { testGetProofs(t, 1) }
-
 func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) }
 
 func testGetProofs(t *testing.T, protocol int) {
@@ -389,27 +366,126 @@ func testGetProofs(t *testing.T, protocol int) {
 	case 2:
 		cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs))
 		sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs)
-		msg, err := peer.app.ReadMsg()
-		if err != nil {
-			t.Errorf("Message read error: %v", err)
+		if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil {
+			t.Errorf("proofs mismatch: %v", err)
 		}
-		var resp struct {
-			ReqID, BV uint64
-			Data      light.NodeList
+	}
+}
+
+// Tests that CHT proofs can be correctly retrieved.
+func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) }
+func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) }
+
+func testGetCHTProofs(t *testing.T, protocol int) {
+	// Figure out the client's CHT frequency
+	frequency := uint64(light.CHTFrequencyClient)
+	if protocol == 1 {
+		frequency = uint64(light.CHTFrequencyServer)
+	}
+	// Assemble the test environment
+	db, _ := ethdb.NewMemDatabase()
+	pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db)
+	bc := pm.blockchain.(*core.BlockChain)
+	peer, _ := newTestPeer(t, "peer", protocol, pm, true)
+	defer peer.close()
+
+	// Wait a while for the CHT indexer to process the new headers
+	time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling
+	time.Sleep(250 * time.Millisecond)                                                     // CI tester slack
+
+	// Assemble the proofs from the different protocols
+	header := bc.GetHeaderByNumber(frequency)
+	rlp, _ := rlp.EncodeToBytes(header)
+
+	key := make([]byte, 8)
+	binary.BigEndian.PutUint64(key, frequency)
+
+	proofsV1 := []ChtResp{{
+		Header: header,
+	}}
+	proofsV2 := HelperTrieResps{
+		AuxData: [][]byte{rlp},
+	}
+	switch protocol {
+	case 1:
+		root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
+		trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))
+
+		var proof light.NodeList
+		trie.Prove(key, 0, &proof)
+		proofsV1[0].Proof = proof
+
+	case 2:
+		root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
+		trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))
+		trie.Prove(key, 0, &proofsV2.Proofs)
+	}
+	// Assemble the requests for the different protocols
+	requestsV1 := []ChtReq{{
+		ChtNum:   1,
+		BlockNum: frequency,
+	}}
+	requestsV2 := []HelperTrieReq{{
+		Type:    htCanonical,
+		TrieIdx: 0,
+		Key:     key,
+		AuxReq:  auxHeader,
+	}}
+	// Send the proof request and verify the response
+	switch protocol {
+	case 1:
+		cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1))
+		sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1)
+		if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil {
+			t.Errorf("proofs mismatch: %v", err)
 		}
-		if err := msg.Decode(&resp); err != nil {
-			t.Errorf("reply decode error: %v", err)
+	case 2:
+		cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2))
+		sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2)
+		if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil {
+			t.Errorf("proofs mismatch: %v", err)
 		}
-		if msg.Code != ProofsV2Msg {
-			t.Errorf("Message code mismatch")
+	}
+}
+
+// Tests that bloombits proofs can be correctly retrieved.
+func TestGetBloombitsProofs(t *testing.T) {
+	// Assemble the test environment
+	db, _ := ethdb.NewMemDatabase()
+	pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db)
+	bc := pm.blockchain.(*core.BlockChain)
+	peer, _ := newTestPeer(t, "peer", 2, pm, true)
+	defer peer.close()
+
+	// Wait a while for the bloombits indexer to process the new headers
+	time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling
+	time.Sleep(250 * time.Millisecond)                                                // CI tester slack
+
+	// Request and verify each bit of the bloom bits proofs
+	for bit := 0; bit < 2048; bit++ {
+		// Assemble therequest and proofs for the bloombits
+		key := make([]byte, 10)
+
+		binary.BigEndian.PutUint16(key[:2], uint16(bit))
+		binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency))
+
+		requests := []HelperTrieReq{{
+			Type:    htBloomBits,
+			TrieIdx: 0,
+			Key:     key,
+		}}
+		var proofs HelperTrieResps
+
+		root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash())
+		trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix)))
+		trie.Prove(key, 0, &proofs.Proofs)
+
+		// Send the proof request and verify the response
+		cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests))
+		sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests)
+		if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil {
+			t.Errorf("bit %d: proofs mismatch: %v", bit, err)
 		}
-		if resp.ReqID != 42 {
-			t.Errorf("ReqID mismatch")
-		}
-		if resp.BV != testBufLimit {
-			t.Errorf("BV mismatch")
-		}
-		testCheckProof(t, proofsV2, resp.Data)
 	}
 }
 
diff --git a/les/helper_test.go b/les/helper_test.go
index bf08e1e2f7..6d997a1a36 100644
--- a/les/helper_test.go
+++ b/les/helper_test.go
@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/les/flowcontrol"
@@ -55,6 +56,9 @@ var (
 	testContractCodeDeployed = testContractCode[16:]
 	testContractDeployed     = uint64(2)
 
+	testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029")
+	testEventEmitterAddr common.Address
+
 	testBufLimit = uint64(100)
 )
 
@@ -85,15 +89,19 @@ func testChainGen(i int, block *core.BlockGen) {
 		// In block 2, the test bank sends some more ether to account #1.
 		// acc1Addr passes it on to account #2.
 		// acc1Addr creates a test contract.
-		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
+		// acc1Addr creates a test event.
 		nonce := block.TxNonce(acc1Addr)
+
+		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
 		tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key)
-		nonce++
-		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
-		testContractAddr = crypto.CreateAddress(acc1Addr, nonce)
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
+		testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1)
+		tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key)
+		testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2)
 		block.AddTx(tx1)
 		block.AddTx(tx2)
 		block.AddTx(tx3)
+		block.AddTx(tx4)
 	case 2:
 		// Block 3 is empty but was mined by account #2.
 		block.SetCoinbase(acc2Addr)
@@ -147,6 +155,16 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor
 		chain, _ = light.NewLightChain(odr, gspec.Config, engine)
 	} else {
 		blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
+
+		chtIndexer := light.NewChtIndexer(db, false)
+		chtIndexer.Start(blockchain)
+
+		bbtIndexer := light.NewBloomTrieIndexer(db, false)
+
+		bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks)
+		bloomIndexer.AddChildIndexer(bbtIndexer)
+		bloomIndexer.Start(blockchain)
+
 		gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator)
 		if _, err := blockchain.InsertChain(gchain); err != nil {
 			panic(err)
diff --git a/les/metrics.go b/les/metrics.go
index 0162a1d1ad..c282a62a1a 100644
--- a/les/metrics.go
+++ b/les/metrics.go
@@ -58,10 +58,10 @@ var (
 		reqReceiptInTrafficMeter  = metrics.NewMeter("eth/req/receipts/in/traffic")
 		reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets")
 		reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")*/
-	miscInPacketsMeter  = metrics.NewMeter("les/misc/in/packets")
-	miscInTrafficMeter  = metrics.NewMeter("les/misc/in/traffic")
-	miscOutPacketsMeter = metrics.NewMeter("les/misc/out/packets")
-	miscOutTrafficMeter = metrics.NewMeter("les/misc/out/traffic")
+	miscInPacketsMeter  = metrics.NewRegisteredMeter("les/misc/in/packets", nil)
+	miscInTrafficMeter  = metrics.NewRegisteredMeter("les/misc/in/traffic", nil)
+	miscOutPacketsMeter = metrics.NewRegisteredMeter("les/misc/out/packets", nil)
+	miscOutTrafficMeter = metrics.NewRegisteredMeter("les/misc/out/traffic", nil)
 )
 
 // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of
diff --git a/les/odr_requests.go b/les/odr_requests.go
index 937a4f1d9d..34d759dd2a 100644
--- a/les/odr_requests.go
+++ b/les/odr_requests.go
@@ -321,7 +321,7 @@ const (
 )
 
 type HelperTrieReq struct {
-	HelperTrieType    uint
+	Type              uint
 	TrieIdx           uint64
 	Key               []byte
 	FromLevel, AuxReq uint
@@ -365,7 +365,7 @@ func (r *ChtRequest) CanSend(peer *peer) bool {
 	peer.lock.RLock()
 	defer peer.lock.RUnlock()
 
-	return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.ChtFrequency
+	return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.CHTFrequencyClient
 }
 
 // Request sends an ODR request to the LES network (implementation of LesOdrRequest)
@@ -374,10 +374,10 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error {
 	var encNum [8]byte
 	binary.BigEndian.PutUint64(encNum[:], r.BlockNum)
 	req := HelperTrieReq{
-		HelperTrieType: htCanonical,
-		TrieIdx:        r.ChtNum,
-		Key:            encNum[:],
-		AuxReq:         auxHeader,
+		Type:    htCanonical,
+		TrieIdx: r.ChtNum,
+		Key:     encNum[:],
+		AuxReq:  auxHeader,
 	}
 	return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req})
 }
@@ -493,14 +493,14 @@ func (r *BloomRequest) Request(reqID uint64, peer *peer) error {
 	reqs := make([]HelperTrieReq, len(r.SectionIdxList))
 
 	var encNumber [10]byte
-	binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx))
+	binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx))
 
 	for i, sectionIdx := range r.SectionIdxList {
-		binary.BigEndian.PutUint64(encNumber[2:10], sectionIdx)
+		binary.BigEndian.PutUint64(encNumber[2:], sectionIdx)
 		reqs[i] = HelperTrieReq{
-			HelperTrieType: htBloomBits,
-			TrieIdx:        r.BloomTrieNum,
-			Key:            common.CopyBytes(encNumber[:]),
+			Type:    htBloomBits,
+			TrieIdx: r.BloomTrieNum,
+			Key:     common.CopyBytes(encNumber[:]),
 		}
 	}
 	return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), reqs)
@@ -525,10 +525,10 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error {
 
 	// Verify the proofs
 	var encNumber [10]byte
-	binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx))
+	binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx))
 
 	for i, idx := range r.SectionIdxList {
-		binary.BigEndian.PutUint64(encNumber[2:10], idx)
+		binary.BigEndian.PutUint64(encNumber[2:], idx)
 		value, err, _ := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads)
 		if err != nil {
 			return err
diff --git a/les/peer.go b/les/peer.go
index b72c80d35a..caf5680778 100644
--- a/les/peer.go
+++ b/les/peer.go
@@ -281,7 +281,6 @@ func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error {
 	default:
 		panic(nil)
 	}
-
 }
 
 // RequestHelperTrieProofs fetches a batch of HelperTrie merkle proofs from a remote node.
@@ -291,12 +290,12 @@ func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, reqs []HelperTrieReq)
 	case lpv1:
 		reqsV1 := make([]ChtReq, len(reqs))
 		for i, req := range reqs {
-			if req.HelperTrieType != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 {
+			if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 {
 				return fmt.Errorf("Request invalid in LES/1 mode")
 			}
 			blockNum := binary.BigEndian.Uint64(req.Key)
 			// convert HelperTrie request to old CHT request
-			reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.ChtFrequency / light.ChtV1Frequency), BlockNum: blockNum, FromLevel: req.FromLevel}
+			reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.CHTFrequencyClient / light.CHTFrequencyServer), BlockNum: blockNum, FromLevel: req.FromLevel}
 		}
 		return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqsV1)
 	case lpv2:
diff --git a/les/retrieve.go b/les/retrieve.go
index dd15b56acf..e262a3cb47 100644
--- a/les/retrieve.go
+++ b/les/retrieve.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
diff --git a/les/server.go b/les/server.go
index 85ebbf8988..28b87008a0 100644
--- a/les/server.go
+++ b/les/server.go
@@ -20,7 +20,6 @@ package les
 import (
 	"crypto/ecdsa"
 	"encoding/binary"
-	"fmt"
 	"math"
 	"sync"
 
@@ -73,23 +72,22 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) {
 	logger := log.New()
 
 	chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility
-	chtV2SectionCount := chtV1SectionCount / (light.ChtFrequency / light.ChtV1Frequency)
+	chtV2SectionCount := chtV1SectionCount / (light.CHTFrequencyClient / light.CHTFrequencyServer)
 	if chtV2SectionCount != 0 {
 		// convert to LES/2 section
 		chtLastSection := chtV2SectionCount - 1
 		// convert last LES/2 section index back to LES/1 index for chtIndexer.SectionHead
-		chtLastSectionV1 := (chtLastSection+1)*(light.ChtFrequency/light.ChtV1Frequency) - 1
+		chtLastSectionV1 := (chtLastSection+1)*(light.CHTFrequencyClient/light.CHTFrequencyServer) - 1
 		chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1)
 		chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead)
-		logger.Info("CHT", "section", chtLastSection, "sectionHead", fmt.Sprintf("%064x", chtSectionHead), "root", fmt.Sprintf("%064x", chtRoot))
+		logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot)
 	}
-
 	bloomTrieSectionCount, _, _ := srv.bloomTrieIndexer.Sections()
 	if bloomTrieSectionCount != 0 {
 		bloomTrieLastSection := bloomTrieSectionCount - 1
 		bloomTrieSectionHead := srv.bloomTrieIndexer.SectionHead(bloomTrieLastSection)
 		bloomTrieRoot := light.GetBloomTrieRoot(pm.chainDb, bloomTrieLastSection, bloomTrieSectionHead)
-		logger.Info("BloomTrie", "section", bloomTrieLastSection, "sectionHead", fmt.Sprintf("%064x", bloomTrieSectionHead), "root", fmt.Sprintf("%064x", bloomTrieRoot))
+		logger.Info("Loaded bloom trie", "section", bloomTrieLastSection, "head", bloomTrieSectionHead, "root", bloomTrieRoot)
 	}
 
 	srv.chtIndexer.Start(eth.BlockChain())
@@ -111,15 +109,17 @@ func (s *LesServer) Protocols() []p2p.Protocol {
 // Start starts the LES server
 func (s *LesServer) Start(srvr *p2p.Server) {
 	s.protocolManager.Start(s.config.LightPeers)
-	for _, topic := range s.lesTopics {
-		topic := topic
-		go func() {
-			logger := log.New("topic", topic)
-			logger.Info("Starting topic registration")
-			defer logger.Info("Terminated topic registration")
+	if srvr.DiscV5 != nil {
+		for _, topic := range s.lesTopics {
+			topic := topic
+			go func() {
+				logger := log.New("topic", topic)
+				logger.Info("Starting topic registration")
+				defer logger.Info("Terminated topic registration")
 
-			srvr.DiscV5.RegisterTopic(topic, s.quitSync)
-		}()
+				srvr.DiscV5.RegisterTopic(topic, s.quitSync)
+			}()
+		}
 	}
 	s.privateKey = srvr.PrivateKey
 	s.protocolManager.blockLoop()
diff --git a/light/lightchain.go b/light/lightchain.go
index 24529ef82e..2784615d35 100644
--- a/light/lightchain.go
+++ b/light/lightchain.go
@@ -100,7 +100,6 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
 	if cp, ok := trustedCheckpoints[bc.genesisBlock.Hash()]; ok {
 		bc.addTrustedCheckpoint(cp)
 	}
-
 	if err := bc.loadLastState(); err != nil {
 		return nil, err
 	}
@@ -128,7 +127,7 @@ func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) {
 	if self.odr.BloomIndexer() != nil {
 		self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead)
 	}
-	log.Info("Added trusted checkpoint", "chain name", cp.name)
+	log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*CHTFrequencyClient-1, "hash", cp.sectionHead)
 }
 
 func (self *LightChain) getProcInterrupt() bool {
@@ -172,9 +171,6 @@ func (bc *LightChain) SetHead(head uint64) {
 
 // GasLimit returns the gas limit of the current HEAD block.
 func (self *LightChain) GasLimit() uint64 {
-	self.mu.RLock()
-	defer self.mu.RUnlock()
-
 	return self.hc.CurrentHeader().GasLimit
 }
 
@@ -388,9 +384,6 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int)
 // CurrentHeader retrieves the current head header of the canonical chain. The
 // header is retrieved from the HeaderChain's internal cache.
 func (self *LightChain) CurrentHeader() *types.Header {
-	self.mu.RLock()
-	defer self.mu.RUnlock()
-
 	return self.hc.CurrentHeader()
 }
 
@@ -454,8 +447,8 @@ func (self *LightChain) SyncCht(ctx context.Context) bool {
 	}
 	headNum := self.CurrentHeader().Number.Uint64()
 	chtCount, _, _ := self.odr.ChtIndexer().Sections()
-	if headNum+1 < chtCount*ChtFrequency {
-		num := chtCount*ChtFrequency - 1
+	if headNum+1 < chtCount*CHTFrequencyClient {
+		num := chtCount*CHTFrequencyClient - 1
 		header, err := GetHeaderByNumber(ctx, self.odr, num)
 		if header != nil && err == nil {
 			self.mu.Lock()
diff --git a/light/nodeset.go b/light/nodeset.go
index ffdb71bb79..6f25219c13 100644
--- a/light/nodeset.go
+++ b/light/nodeset.go
@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -29,7 +29,9 @@ import (
 // NodeSet stores a set of trie nodes. It implements trie.Database and can also
 // act as a cache for another trie.Database.
 type NodeSet struct {
-	db       map[string][]byte
+	nodes map[string][]byte
+	order []string
+
 	dataSize int
 	lock     sync.RWMutex
 }
@@ -37,7 +39,7 @@ type NodeSet struct {
 // NewNodeSet creates an empty node set
 func NewNodeSet() *NodeSet {
 	return &NodeSet{
-		db: make(map[string][]byte),
+		nodes: make(map[string][]byte),
 	}
 }
 
@@ -46,10 +48,15 @@ func (db *NodeSet) Put(key []byte, value []byte) error {
 	db.lock.Lock()
 	defer db.lock.Unlock()
 
-	if _, ok := db.db[string(key)]; !ok {
-		db.db[string(key)] = common.CopyBytes(value)
-		db.dataSize += len(value)
+	if _, ok := db.nodes[string(key)]; ok {
+		return nil
 	}
+	keystr := string(key)
+
+	db.nodes[keystr] = common.CopyBytes(value)
+	db.order = append(db.order, keystr)
+	db.dataSize += len(value)
+
 	return nil
 }
 
@@ -58,7 +65,7 @@ func (db *NodeSet) Get(key []byte) ([]byte, error) {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
-	if entry, ok := db.db[string(key)]; ok {
+	if entry, ok := db.nodes[string(key)]; ok {
 		return entry, nil
 	}
 	return nil, errors.New("not found")
@@ -75,7 +82,7 @@ func (db *NodeSet) KeyCount() int {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
-	return len(db.db)
+	return len(db.nodes)
 }
 
 // DataSize returns the aggregated data size of nodes in the set
@@ -92,8 +99,8 @@ func (db *NodeSet) NodeList() NodeList {
 	defer db.lock.RUnlock()
 
 	var values NodeList
-	for _, value := range db.db {
-		values = append(values, value)
+	for _, key := range db.order {
+		values = append(values, db.nodes[key])
 	}
 	return values
 }
@@ -103,7 +110,7 @@ func (db *NodeSet) Store(target ethdb.Putter) {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
-	for key, value := range db.db {
+	for key, value := range db.nodes {
 		target.Put([]byte(key), value)
 	}
 }
diff --git a/light/odr_util.go b/light/odr_util.go
index a0eb6303d4..97ba440ace 100644
--- a/light/odr_util.go
+++ b/light/odr_util.go
@@ -52,23 +52,20 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ
 		for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
 			chtCount--
 			if chtCount > 0 {
-				sectionHeadNum = chtCount*ChtFrequency - 1
+				sectionHeadNum = chtCount*CHTFrequencyClient - 1
 				sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1)
 				canonicalHash = core.GetCanonicalHash(db, sectionHeadNum)
 			}
 		}
 	}
-
-	if number >= chtCount*ChtFrequency {
+	if number >= chtCount*CHTFrequencyClient {
 		return nil, ErrNoTrustedCht
 	}
-
 	r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number}
 	if err := odr.Retrieve(ctx, r); err != nil {
 		return nil, err
-	} else {
-		return r.Header, nil
 	}
+	return r.Header, nil
 }
 
 func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) {
@@ -129,15 +126,48 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint
 // GetBlockReceipts retrieves the receipts generated by the transactions included
 // in a block given by its hash.
 func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) {
+	// Retrieve the potentially incomplete receipts from disk or network
 	receipts := core.GetBlockReceipts(odr.Database(), hash, number)
-	if receipts != nil {
-		return receipts, nil
+	if receipts == nil {
+		r := &ReceiptsRequest{Hash: hash, Number: number}
+		if err := odr.Retrieve(ctx, r); err != nil {
+			return nil, err
+		}
+		receipts = r.Receipts
 	}
-	r := &ReceiptsRequest{Hash: hash, Number: number}
-	if err := odr.Retrieve(ctx, r); err != nil {
-		return nil, err
+	// If the receipts are incomplete, fill the derived fields
+	if len(receipts) > 0 && receipts[0].TxHash == (common.Hash{}) {
+		block, err := GetBlock(ctx, odr, hash, number)
+		if err != nil {
+			return nil, err
+		}
+		genesis := core.GetCanonicalHash(odr.Database(), 0)
+		config, _ := core.GetChainConfig(odr.Database(), genesis)
+
+		core.SetReceiptsData(config, block, receipts)
+		core.WriteBlockReceipts(odr.Database(), hash, number, receipts)
 	}
-	return r.Receipts, nil
+	return receipts, nil
+}
+
+// GetBlockLogs retrieves the logs generated by the transactions included in a
+// block given by its hash.
+func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) {
+	// Retrieve the potentially incomplete receipts from disk or network
+	receipts := core.GetBlockReceipts(odr.Database(), hash, number)
+	if receipts == nil {
+		r := &ReceiptsRequest{Hash: hash, Number: number}
+		if err := odr.Retrieve(ctx, r); err != nil {
+			return nil, err
+		}
+		receipts = r.Receipts
+	}
+	// Return the logs without deriving any computed fields on the receipts
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
 }
 
 // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes
diff --git a/light/postprocess.go b/light/postprocess.go
index bbac58d121..84149fdaae 100644
--- a/light/postprocess.go
+++ b/light/postprocess.go
@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -19,7 +19,6 @@ package light
 import (
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"math/big"
 	"time"
 
@@ -35,8 +34,14 @@ import (
 )
 
 const (
-	ChtFrequency                   = 32768
-	ChtV1Frequency                 = 4096 // as long as we want to retain LES/1 compatibility, servers generate CHTs with the old, higher frequency
+	// CHTFrequencyClient is the block frequency for creating CHTs on the client side.
+	CHTFrequencyClient = 32768
+
+	// CHTFrequencyServer is the block frequency for creating CHTs on the server side.
+	// Eventually this can be merged back with the client version, but that requires a
+	// full database upgrade, so that should be left for a suitable moment.
+	CHTFrequencyServer = 4096
+
 	HelperTrieConfirmations        = 2048 // number of confirmations before a server is expected to have the given HelperTrie available
 	HelperTrieProcessConfirmations = 256  // number of confirmations before a HelperTrie is generated
 )
@@ -52,19 +57,19 @@ type trustedCheckpoint struct {
 
 var (
 	mainnetCheckpoint = trustedCheckpoint{
-		name:          "ETH mainnet",
-		sectionIdx:    150,
-		sectionHead:   common.HexToHash("1e2e67f289565cbe7bd4367f7960dbd73a3f7c53439e1047cd7ba331c8109e39"),
-		chtRoot:       common.HexToHash("f2a6c9ca143d647b44523cc249f1072c8912358ab873a77a5fdc792b8df99e80"),
-		bloomTrieRoot: common.HexToHash("c018952fa1513c97857e79fbb9a37acaf8432d5b85e52a78eca7dff5fd5900ee"),
+		name:          "mainnet",
+		sectionIdx:    153,
+		sectionHead:   common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"),
+		chtRoot:       common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"),
+		bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"),
 	}
 
 	ropstenCheckpoint = trustedCheckpoint{
-		name:          "Ropsten testnet",
-		sectionIdx:    75,
-		sectionHead:   common.HexToHash("12e68324f4578ea3e8e7fb3968167686729396c9279287fa1f1a8b51bb2d05b4"),
-		chtRoot:       common.HexToHash("3e51dc095c69fa654a4cac766e0afff7357515b4b3c3a379c675f810363e54be"),
-		bloomTrieRoot: common.HexToHash("33e3a70b33c1d73aa698d496a80615e98ed31fa8f56969876180553b32333339"),
+		name:          "ropsten",
+		sectionIdx:    79,
+		sectionHead:   common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"),
+		chtRoot:       common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"),
+		bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"),
 	}
 )
 
@@ -100,7 +105,7 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c
 // GetChtV2Root reads the CHT root assoctiated to the given section from the database
 // Note that sectionIdx is specified according to LES/2 CHT section size
 func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
-	return GetChtRoot(db, (sectionIdx+1)*(ChtFrequency/ChtV1Frequency)-1, sectionHead)
+	return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead)
 }
 
 // StoreChtRoot writes the CHT root assoctiated to the given section into the database
@@ -124,10 +129,10 @@ type ChtIndexerBackend struct {
 func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer {
 	var sectionSize, confirmReq uint64
 	if clientMode {
-		sectionSize = ChtFrequency
+		sectionSize = CHTFrequencyClient
 		confirmReq = HelperTrieConfirmations
 	} else {
-		sectionSize = ChtV1Frequency
+		sectionSize = CHTFrequencyServer
 		confirmReq = HelperTrieProcessConfirmations
 	}
 	idb := ethdb.NewTable(db, "chtIndex-")
@@ -174,8 +179,8 @@ func (c *ChtIndexerBackend) Commit() error {
 	}
 	c.triedb.Commit(root, false)
 
-	if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 {
-		log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root))
+	if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 {
+		log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root)
 	}
 	StoreChtRoot(c.diskdb, c.section, c.lastHash, root)
 	return nil
@@ -294,7 +299,7 @@ func (b *BloomTrieIndexerBackend) Commit() error {
 	b.triedb.Commit(root, false)
 
 	sectionHead := b.sectionHeads[b.bloomTrieRatio-1]
-	log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize))
+	log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize))
 	StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root)
 
 	return nil
diff --git a/metrics/FORK.md b/metrics/FORK.md
new file mode 100644
index 0000000000..b19985bf56
--- /dev/null
+++ b/metrics/FORK.md
@@ -0,0 +1 @@
+This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09
diff --git a/vendor/github.com/rcrowley/go-metrics/LICENSE b/metrics/LICENSE
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/LICENSE
rename to metrics/LICENSE
diff --git a/vendor/github.com/rcrowley/go-metrics/README.md b/metrics/README.md
similarity index 81%
rename from vendor/github.com/rcrowley/go-metrics/README.md
rename to metrics/README.md
index 2d1a6dcfa4..bc2a45a838 100644
--- a/vendor/github.com/rcrowley/go-metrics/README.md
+++ b/metrics/README.md
@@ -42,12 +42,22 @@ t.Update(47)
 Register() is not threadsafe. For threadsafe metric registration use
 GetOrRegister:
 
-```
+```go
 t := metrics.GetOrRegisterTimer("account.create.latency", nil)
 t.Time(func() {})
 t.Update(47)
 ```
 
+**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will
+leak memory:
+
+```go
+// Will call Stop() on the Meter to allow for garbage collection
+metrics.Unregister("quux")
+// Or similarly for a Timer that embeds a Meter
+metrics.Unregister("bang")
+```
+
 Periodically log every metric in human-readable form to standard error:
 
 ```go
@@ -81,12 +91,13 @@ issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and
 ```go
 import "github.com/vrischmann/go-metrics-influxdb"
 
-go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{
-    Host:     "127.0.0.1:8086",
-    Database: "metrics",
-    Username: "test",
-    Password: "test",
-})
+go influxdb.InfluxDB(metrics.DefaultRegistry,
+  10e9, 
+  "127.0.0.1:8086", 
+  "database-name", 
+  "username", 
+  "password"
+)
 ```
 
 Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato):
@@ -146,8 +157,10 @@ Publishing Metrics
 
 Clients are available for the following destinations:
 
-* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato)
-* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite)
-* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb)
-* Ganglia - [https://github.com/appscode/metlia](https://github.com/appscode/metlia)
-* Prometheus - [https://github.com/deathowl/go-metrics-prometheus](https://github.com/deathowl/go-metrics-prometheus)
+* Librato - https://github.com/mihasya/go-metrics-librato
+* Graphite - https://github.com/cyberdelia/go-metrics-graphite
+* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb
+* Ganglia - https://github.com/appscode/metlia
+* Prometheus - https://github.com/deathowl/go-metrics-prometheus
+* DataDog - https://github.com/syntaqx/go-metrics-datadog
+* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx
diff --git a/vendor/github.com/rcrowley/go-metrics/counter.go b/metrics/counter.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/counter.go
rename to metrics/counter.go
index bb7b039cb5..c7f2b4bd3a 100644
--- a/vendor/github.com/rcrowley/go-metrics/counter.go
+++ b/metrics/counter.go
@@ -22,7 +22,7 @@ func GetOrRegisterCounter(name string, r Registry) Counter {
 
 // NewCounter constructs a new StandardCounter.
 func NewCounter() Counter {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilCounter{}
 	}
 	return &StandardCounter{0}
diff --git a/metrics/counter_test.go b/metrics/counter_test.go
new file mode 100644
index 0000000000..dfb03b4e88
--- /dev/null
+++ b/metrics/counter_test.go
@@ -0,0 +1,77 @@
+package metrics
+
+import "testing"
+
+func BenchmarkCounter(b *testing.B) {
+	c := NewCounter()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		c.Inc(1)
+	}
+}
+
+func TestCounterClear(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	c.Clear()
+	if count := c.Count(); 0 != count {
+		t.Errorf("c.Count(): 0 != %v\n", count)
+	}
+}
+
+func TestCounterDec1(t *testing.T) {
+	c := NewCounter()
+	c.Dec(1)
+	if count := c.Count(); -1 != count {
+		t.Errorf("c.Count(): -1 != %v\n", count)
+	}
+}
+
+func TestCounterDec2(t *testing.T) {
+	c := NewCounter()
+	c.Dec(2)
+	if count := c.Count(); -2 != count {
+		t.Errorf("c.Count(): -2 != %v\n", count)
+	}
+}
+
+func TestCounterInc1(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	if count := c.Count(); 1 != count {
+		t.Errorf("c.Count(): 1 != %v\n", count)
+	}
+}
+
+func TestCounterInc2(t *testing.T) {
+	c := NewCounter()
+	c.Inc(2)
+	if count := c.Count(); 2 != count {
+		t.Errorf("c.Count(): 2 != %v\n", count)
+	}
+}
+
+func TestCounterSnapshot(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	snapshot := c.Snapshot()
+	c.Inc(1)
+	if count := snapshot.Count(); 1 != count {
+		t.Errorf("c.Count(): 1 != %v\n", count)
+	}
+}
+
+func TestCounterZero(t *testing.T) {
+	c := NewCounter()
+	if count := c.Count(); 0 != count {
+		t.Errorf("c.Count(): 0 != %v\n", count)
+	}
+}
+
+func TestGetOrRegisterCounter(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredCounter("foo", r).Inc(47)
+	if c := GetOrRegisterCounter("foo", r); 47 != c.Count() {
+		t.Fatal(c)
+	}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/debug.go b/metrics/debug.go
similarity index 94%
rename from vendor/github.com/rcrowley/go-metrics/debug.go
rename to metrics/debug.go
index 043ccefab6..de4a2739fe 100644
--- a/vendor/github.com/rcrowley/go-metrics/debug.go
+++ b/metrics/debug.go
@@ -22,7 +22,7 @@ var (
 // Capture new values for the Go garbage collector statistics exported in
 // debug.GCStats.  This is designed to be called as a goroutine.
 func CaptureDebugGCStats(r Registry, d time.Duration) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		CaptureDebugGCStatsOnce(r)
 	}
 }
@@ -41,8 +41,8 @@ func CaptureDebugGCStatsOnce(r Registry) {
 	debug.ReadGCStats(&gcStats)
 	debugMetrics.ReadGCStats.UpdateSince(t)
 
-	debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano()))
-	debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC))
+	debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano())
+	debugMetrics.GCStats.NumGC.Update(gcStats.NumGC)
 	if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) {
 		debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0]))
 	}
diff --git a/metrics/debug_test.go b/metrics/debug_test.go
new file mode 100644
index 0000000000..07eb867841
--- /dev/null
+++ b/metrics/debug_test.go
@@ -0,0 +1,48 @@
+package metrics
+
+import (
+	"runtime"
+	"runtime/debug"
+	"testing"
+	"time"
+)
+
+func BenchmarkDebugGCStats(b *testing.B) {
+	r := NewRegistry()
+	RegisterDebugGCStats(r)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		CaptureDebugGCStatsOnce(r)
+	}
+}
+
+func TestDebugGCStatsBlocking(t *testing.T) {
+	if g := runtime.GOMAXPROCS(0); g < 2 {
+		t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g)
+		return
+	}
+	ch := make(chan int)
+	go testDebugGCStatsBlocking(ch)
+	var gcStats debug.GCStats
+	t0 := time.Now()
+	debug.ReadGCStats(&gcStats)
+	t1 := time.Now()
+	t.Log("i++ during debug.ReadGCStats:", <-ch)
+	go testDebugGCStatsBlocking(ch)
+	d := t1.Sub(t0)
+	t.Log(d)
+	time.Sleep(d)
+	t.Log("i++ during time.Sleep:", <-ch)
+}
+
+func testDebugGCStatsBlocking(ch chan int) {
+	i := 0
+	for {
+		select {
+		case ch <- i:
+			return
+		default:
+			i++
+		}
+	}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/ewma.go b/metrics/ewma.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/ewma.go
rename to metrics/ewma.go
index 694a1d0330..3aecd4fa35 100644
--- a/vendor/github.com/rcrowley/go-metrics/ewma.go
+++ b/metrics/ewma.go
@@ -17,7 +17,7 @@ type EWMA interface {
 
 // NewEWMA constructs a new EWMA with the given alpha.
 func NewEWMA(alpha float64) EWMA {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilEWMA{}
 	}
 	return &StandardEWMA{alpha: alpha}
diff --git a/metrics/ewma_test.go b/metrics/ewma_test.go
new file mode 100644
index 0000000000..0430fbd247
--- /dev/null
+++ b/metrics/ewma_test.go
@@ -0,0 +1,225 @@
+package metrics
+
+import "testing"
+
+func BenchmarkEWMA(b *testing.B) {
+	a := NewEWMA1()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		a.Update(1)
+		a.Tick()
+	}
+}
+
+func TestEWMA1(t *testing.T) {
+	a := NewEWMA1()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.22072766470286553 != rate {
+		t.Errorf("1 minute a.Rate(): 0.22072766470286553 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.08120116994196772 != rate {
+		t.Errorf("2 minute a.Rate(): 0.08120116994196772 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.029872241020718428 != rate {
+		t.Errorf("3 minute a.Rate(): 0.029872241020718428 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.01098938333324054 != rate {
+		t.Errorf("4 minute a.Rate(): 0.01098938333324054 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.004042768199451294 != rate {
+		t.Errorf("5 minute a.Rate(): 0.004042768199451294 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0014872513059998212 != rate {
+		t.Errorf("6 minute a.Rate(): 0.0014872513059998212 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0005471291793327122 != rate {
+		t.Errorf("7 minute a.Rate(): 0.0005471291793327122 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.00020127757674150815 != rate {
+		t.Errorf("8 minute a.Rate(): 0.00020127757674150815 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 7.404588245200814e-05 != rate {
+		t.Errorf("9 minute a.Rate(): 7.404588245200814e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 2.7239957857491083e-05 != rate {
+		t.Errorf("10 minute a.Rate(): 2.7239957857491083e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.0021020474147462e-05 != rate {
+		t.Errorf("11 minute a.Rate(): 1.0021020474147462e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 3.6865274119969525e-06 != rate {
+		t.Errorf("12 minute a.Rate(): 3.6865274119969525e-06 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.3561976441886433e-06 != rate {
+		t.Errorf("13 minute a.Rate(): 1.3561976441886433e-06 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 4.989172314621449e-07 != rate {
+		t.Errorf("14 minute a.Rate(): 4.989172314621449e-07 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.8354139230109722e-07 != rate {
+		t.Errorf("15 minute a.Rate(): 1.8354139230109722e-07 != %v\n", rate)
+	}
+}
+
+func TestEWMA5(t *testing.T) {
+	a := NewEWMA5()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.49123845184678905 != rate {
+		t.Errorf("1 minute a.Rate(): 0.49123845184678905 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4021920276213837 != rate {
+		t.Errorf("2 minute a.Rate(): 0.4021920276213837 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.32928698165641596 != rate {
+		t.Errorf("3 minute a.Rate(): 0.32928698165641596 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.269597378470333 != rate {
+		t.Errorf("4 minute a.Rate(): 0.269597378470333 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2207276647028654 != rate {
+		t.Errorf("5 minute a.Rate(): 0.2207276647028654 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.18071652714732128 != rate {
+		t.Errorf("6 minute a.Rate(): 0.18071652714732128 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.14795817836496392 != rate {
+		t.Errorf("7 minute a.Rate(): 0.14795817836496392 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.12113791079679326 != rate {
+		t.Errorf("8 minute a.Rate(): 0.12113791079679326 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.09917933293295193 != rate {
+		t.Errorf("9 minute a.Rate(): 0.09917933293295193 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.08120116994196763 != rate {
+		t.Errorf("10 minute a.Rate(): 0.08120116994196763 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.06648189501740036 != rate {
+		t.Errorf("11 minute a.Rate(): 0.06648189501740036 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.05443077197364752 != rate {
+		t.Errorf("12 minute a.Rate(): 0.05443077197364752 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.04456414692860035 != rate {
+		t.Errorf("13 minute a.Rate(): 0.04456414692860035 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.03648603757513079 != rate {
+		t.Errorf("14 minute a.Rate(): 0.03648603757513079 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0298722410207183831020718428 != rate {
+		t.Errorf("15 minute a.Rate(): 0.0298722410207183831020718428 != %v\n", rate)
+	}
+}
+
+func TestEWMA15(t *testing.T) {
+	a := NewEWMA15()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.5613041910189706 != rate {
+		t.Errorf("1 minute a.Rate(): 0.5613041910189706 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.5251039914257684 != rate {
+		t.Errorf("2 minute a.Rate(): 0.5251039914257684 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4912384518467888184678905 != rate {
+		t.Errorf("3 minute a.Rate(): 0.4912384518467888184678905 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.459557003018789 != rate {
+		t.Errorf("4 minute a.Rate(): 0.459557003018789 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4299187863442732 != rate {
+		t.Errorf("5 minute a.Rate(): 0.4299187863442732 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4021920276213831 != rate {
+		t.Errorf("6 minute a.Rate(): 0.4021920276213831 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.37625345116383313 != rate {
+		t.Errorf("7 minute a.Rate(): 0.37625345116383313 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3519877317060185 != rate {
+		t.Errorf("8 minute a.Rate(): 0.3519877317060185 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3292869816564153165641596 != rate {
+		t.Errorf("9 minute a.Rate(): 0.3292869816564153165641596 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3080502714195546 != rate {
+		t.Errorf("10 minute a.Rate(): 0.3080502714195546 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2881831806538789 != rate {
+		t.Errorf("11 minute a.Rate(): 0.2881831806538789 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.26959737847033216 != rate {
+		t.Errorf("12 minute a.Rate(): 0.26959737847033216 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2522102307052083 != rate {
+		t.Errorf("13 minute a.Rate(): 0.2522102307052083 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.23594443252115815 != rate {
+		t.Errorf("14 minute a.Rate(): 0.23594443252115815 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2207276647028646247028654470286553 != rate {
+		t.Errorf("15 minute a.Rate(): 0.2207276647028646247028654470286553 != %v\n", rate)
+	}
+}
+
+func elapseMinute(a EWMA) {
+	for i := 0; i < 12; i++ {
+		a.Tick()
+	}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/exp/exp.go b/metrics/exp/exp.go
similarity index 73%
rename from vendor/github.com/rcrowley/go-metrics/exp/exp.go
rename to metrics/exp/exp.go
index 11dd3f898a..c19d00a94d 100644
--- a/vendor/github.com/rcrowley/go-metrics/exp/exp.go
+++ b/metrics/exp/exp.go
@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"sync"
 
-	"github.com/rcrowley/go-metrics"
+	"github.com/ethereum/go-ethereum/metrics"
 )
 
 type exp struct {
@@ -97,22 +97,22 @@ func (exp *exp) publishHistogram(name string, metric metrics.Histogram) {
 	exp.getInt(name + ".count").Set(h.Count())
 	exp.getFloat(name + ".min").Set(float64(h.Min()))
 	exp.getFloat(name + ".max").Set(float64(h.Max()))
-	exp.getFloat(name + ".mean").Set(float64(h.Mean()))
-	exp.getFloat(name + ".std-dev").Set(float64(h.StdDev()))
-	exp.getFloat(name + ".50-percentile").Set(float64(ps[0]))
-	exp.getFloat(name + ".75-percentile").Set(float64(ps[1]))
-	exp.getFloat(name + ".95-percentile").Set(float64(ps[2]))
-	exp.getFloat(name + ".99-percentile").Set(float64(ps[3]))
-	exp.getFloat(name + ".999-percentile").Set(float64(ps[4]))
+	exp.getFloat(name + ".mean").Set(h.Mean())
+	exp.getFloat(name + ".std-dev").Set(h.StdDev())
+	exp.getFloat(name + ".50-percentile").Set(ps[0])
+	exp.getFloat(name + ".75-percentile").Set(ps[1])
+	exp.getFloat(name + ".95-percentile").Set(ps[2])
+	exp.getFloat(name + ".99-percentile").Set(ps[3])
+	exp.getFloat(name + ".999-percentile").Set(ps[4])
 }
 
 func (exp *exp) publishMeter(name string, metric metrics.Meter) {
 	m := metric.Snapshot()
 	exp.getInt(name + ".count").Set(m.Count())
-	exp.getFloat(name + ".one-minute").Set(float64(m.Rate1()))
-	exp.getFloat(name + ".five-minute").Set(float64(m.Rate5()))
-	exp.getFloat(name + ".fifteen-minute").Set(float64((m.Rate15())))
-	exp.getFloat(name + ".mean").Set(float64(m.RateMean()))
+	exp.getFloat(name + ".one-minute").Set(m.Rate1())
+	exp.getFloat(name + ".five-minute").Set(m.Rate5())
+	exp.getFloat(name + ".fifteen-minute").Set((m.Rate15()))
+	exp.getFloat(name + ".mean").Set(m.RateMean())
 }
 
 func (exp *exp) publishTimer(name string, metric metrics.Timer) {
@@ -121,17 +121,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) {
 	exp.getInt(name + ".count").Set(t.Count())
 	exp.getFloat(name + ".min").Set(float64(t.Min()))
 	exp.getFloat(name + ".max").Set(float64(t.Max()))
-	exp.getFloat(name + ".mean").Set(float64(t.Mean()))
-	exp.getFloat(name + ".std-dev").Set(float64(t.StdDev()))
-	exp.getFloat(name + ".50-percentile").Set(float64(ps[0]))
-	exp.getFloat(name + ".75-percentile").Set(float64(ps[1]))
-	exp.getFloat(name + ".95-percentile").Set(float64(ps[2]))
-	exp.getFloat(name + ".99-percentile").Set(float64(ps[3]))
-	exp.getFloat(name + ".999-percentile").Set(float64(ps[4]))
-	exp.getFloat(name + ".one-minute").Set(float64(t.Rate1()))
-	exp.getFloat(name + ".five-minute").Set(float64(t.Rate5()))
-	exp.getFloat(name + ".fifteen-minute").Set(float64((t.Rate15())))
-	exp.getFloat(name + ".mean-rate").Set(float64(t.RateMean()))
+	exp.getFloat(name + ".mean").Set(t.Mean())
+	exp.getFloat(name + ".std-dev").Set(t.StdDev())
+	exp.getFloat(name + ".50-percentile").Set(ps[0])
+	exp.getFloat(name + ".75-percentile").Set(ps[1])
+	exp.getFloat(name + ".95-percentile").Set(ps[2])
+	exp.getFloat(name + ".99-percentile").Set(ps[3])
+	exp.getFloat(name + ".999-percentile").Set(ps[4])
+	exp.getFloat(name + ".one-minute").Set(t.Rate1())
+	exp.getFloat(name + ".five-minute").Set(t.Rate5())
+	exp.getFloat(name + ".fifteen-minute").Set(t.Rate15())
+	exp.getFloat(name + ".mean-rate").Set(t.RateMean())
 }
 
 func (exp *exp) syncToExpvar() {
diff --git a/vendor/github.com/rcrowley/go-metrics/gauge.go b/metrics/gauge.go
similarity index 98%
rename from vendor/github.com/rcrowley/go-metrics/gauge.go
rename to metrics/gauge.go
index cb57a93889..0fbfdb8603 100644
--- a/vendor/github.com/rcrowley/go-metrics/gauge.go
+++ b/metrics/gauge.go
@@ -20,7 +20,7 @@ func GetOrRegisterGauge(name string, r Registry) Gauge {
 
 // NewGauge constructs a new StandardGauge.
 func NewGauge() Gauge {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGauge{}
 	}
 	return &StandardGauge{0}
@@ -38,7 +38,7 @@ func NewRegisteredGauge(name string, r Registry) Gauge {
 
 // NewFunctionalGauge constructs a new FunctionalGauge.
 func NewFunctionalGauge(f func() int64) Gauge {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGauge{}
 	}
 	return &FunctionalGauge{value: f}
diff --git a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go b/metrics/gauge_float64.go
similarity index 98%
rename from vendor/github.com/rcrowley/go-metrics/gauge_float64.go
rename to metrics/gauge_float64.go
index 6f93920b2c..66819c9577 100644
--- a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go
+++ b/metrics/gauge_float64.go
@@ -20,7 +20,7 @@ func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 {
 
 // NewGaugeFloat64 constructs a new StandardGaugeFloat64.
 func NewGaugeFloat64() GaugeFloat64 {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGaugeFloat64{}
 	}
 	return &StandardGaugeFloat64{
@@ -40,7 +40,7 @@ func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 {
 
 // NewFunctionalGauge constructs a new FunctionalGauge.
 func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGaugeFloat64{}
 	}
 	return &FunctionalGaugeFloat64{value: f}
diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go
new file mode 100644
index 0000000000..99e62a4030
--- /dev/null
+++ b/metrics/gauge_float64_test.go
@@ -0,0 +1,59 @@
+package metrics
+
+import "testing"
+
+func BenchmarkGuageFloat64(b *testing.B) {
+	g := NewGaugeFloat64()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Update(float64(i))
+	}
+}
+
+func TestGaugeFloat64(t *testing.T) {
+	g := NewGaugeFloat64()
+	g.Update(float64(47.0))
+	if v := g.Value(); float64(47.0) != v {
+		t.Errorf("g.Value(): 47.0 != %v\n", v)
+	}
+}
+
+func TestGaugeFloat64Snapshot(t *testing.T) {
+	g := NewGaugeFloat64()
+	g.Update(float64(47.0))
+	snapshot := g.Snapshot()
+	g.Update(float64(0))
+	if v := snapshot.Value(); float64(47.0) != v {
+		t.Errorf("g.Value(): 47.0 != %v\n", v)
+	}
+}
+
+func TestGetOrRegisterGaugeFloat64(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredGaugeFloat64("foo", r).Update(float64(47.0))
+	t.Logf("registry: %v", r)
+	if g := GetOrRegisterGaugeFloat64("foo", r); float64(47.0) != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func TestFunctionalGaugeFloat64(t *testing.T) {
+	var counter float64
+	fg := NewFunctionalGaugeFloat64(func() float64 {
+		counter++
+		return counter
+	})
+	fg.Value()
+	fg.Value()
+	if counter != 2 {
+		t.Error("counter != 2")
+	}
+}
+
+func TestGetOrRegisterFunctionalGaugeFloat64(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredFunctionalGaugeFloat64("foo", r, func() float64 { return 47 })
+	if g := GetOrRegisterGaugeFloat64("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}
diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go
new file mode 100644
index 0000000000..1f2603d339
--- /dev/null
+++ b/metrics/gauge_test.go
@@ -0,0 +1,68 @@
+package metrics
+
+import (
+	"fmt"
+	"testing"
+)
+
+func BenchmarkGuage(b *testing.B) {
+	g := NewGauge()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Update(int64(i))
+	}
+}
+
+func TestGauge(t *testing.T) {
+	g := NewGauge()
+	g.Update(int64(47))
+	if v := g.Value(); 47 != v {
+		t.Errorf("g.Value(): 47 != %v\n", v)
+	}
+}
+
+func TestGaugeSnapshot(t *testing.T) {
+	g := NewGauge()
+	g.Update(int64(47))
+	snapshot := g.Snapshot()
+	g.Update(int64(0))
+	if v := snapshot.Value(); 47 != v {
+		t.Errorf("g.Value(): 47 != %v\n", v)
+	}
+}
+
+func TestGetOrRegisterGauge(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredGauge("foo", r).Update(47)
+	if g := GetOrRegisterGauge("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func TestFunctionalGauge(t *testing.T) {
+	var counter int64
+	fg := NewFunctionalGauge(func() int64 {
+		counter++
+		return counter
+	})
+	fg.Value()
+	fg.Value()
+	if counter != 2 {
+		t.Error("counter != 2")
+	}
+}
+
+func TestGetOrRegisterFunctionalGauge(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredFunctionalGauge("foo", r, func() int64 { return 47 })
+	if g := GetOrRegisterGauge("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func ExampleGetOrRegisterGauge() {
+	m := "server.bytes_sent"
+	g := GetOrRegisterGauge(m, nil)
+	g.Update(47)
+	fmt.Println(g.Value()) // Output: 47
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/graphite.go b/metrics/graphite.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/graphite.go
rename to metrics/graphite.go
index abd0a7d291..142eec86be 100644
--- a/vendor/github.com/rcrowley/go-metrics/graphite.go
+++ b/metrics/graphite.go
@@ -39,7 +39,7 @@ func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
 // but it takes a GraphiteConfig instead.
 func GraphiteWithConfig(c GraphiteConfig) {
 	log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
-	for _ = range time.Tick(c.FlushInterval) {
+	for range time.Tick(c.FlushInterval) {
 		if err := graphite(&c); nil != err {
 			log.Println(err)
 		}
diff --git a/metrics/graphite_test.go b/metrics/graphite_test.go
new file mode 100644
index 0000000000..c797c781df
--- /dev/null
+++ b/metrics/graphite_test.go
@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"net"
+	"time"
+)
+
+func ExampleGraphite() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go Graphite(DefaultRegistry, 1*time.Second, "some.prefix", addr)
+}
+
+func ExampleGraphiteWithConfig() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go GraphiteWithConfig(GraphiteConfig{
+		Addr:          addr,
+		Registry:      DefaultRegistry,
+		FlushInterval: 1 * time.Second,
+		DurationUnit:  time.Millisecond,
+		Percentiles:   []float64{0.5, 0.75, 0.99, 0.999},
+	})
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/healthcheck.go b/metrics/healthcheck.go
similarity index 98%
rename from vendor/github.com/rcrowley/go-metrics/healthcheck.go
rename to metrics/healthcheck.go
index 445131caee..f1ae31e34a 100644
--- a/vendor/github.com/rcrowley/go-metrics/healthcheck.go
+++ b/metrics/healthcheck.go
@@ -11,7 +11,7 @@ type Healthcheck interface {
 // NewHealthcheck constructs a new Healthcheck which will use the given
 // function to update its status.
 func NewHealthcheck(f func(Healthcheck)) Healthcheck {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilHealthcheck{}
 	}
 	return &StandardHealthcheck{nil, f}
diff --git a/vendor/github.com/rcrowley/go-metrics/histogram.go b/metrics/histogram.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/histogram.go
rename to metrics/histogram.go
index dbc837fe4d..46f3bbd2f1 100644
--- a/vendor/github.com/rcrowley/go-metrics/histogram.go
+++ b/metrics/histogram.go
@@ -28,7 +28,7 @@ func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram {
 
 // NewHistogram constructs a new StandardHistogram from a Sample.
 func NewHistogram(s Sample) Histogram {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilHistogram{}
 	}
 	return &StandardHistogram{sample: s}
diff --git a/metrics/histogram_test.go b/metrics/histogram_test.go
new file mode 100644
index 0000000000..d7f4f0171c
--- /dev/null
+++ b/metrics/histogram_test.go
@@ -0,0 +1,95 @@
+package metrics
+
+import "testing"
+
+func BenchmarkHistogram(b *testing.B) {
+	h := NewHistogram(NewUniformSample(100))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		h.Update(int64(i))
+	}
+}
+
+func TestGetOrRegisterHistogram(t *testing.T) {
+	r := NewRegistry()
+	s := NewUniformSample(100)
+	NewRegisteredHistogram("foo", r, s).Update(47)
+	if h := GetOrRegisterHistogram("foo", r, s); 1 != h.Count() {
+		t.Fatal(h)
+	}
+}
+
+func TestHistogram10000(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	testHistogram10000(t, h)
+}
+
+func TestHistogramEmpty(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100))
+	if count := h.Count(); 0 != count {
+		t.Errorf("h.Count(): 0 != %v\n", count)
+	}
+	if min := h.Min(); 0 != min {
+		t.Errorf("h.Min(): 0 != %v\n", min)
+	}
+	if max := h.Max(); 0 != max {
+		t.Errorf("h.Max(): 0 != %v\n", max)
+	}
+	if mean := h.Mean(); 0.0 != mean {
+		t.Errorf("h.Mean(): 0.0 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 0.0 != stdDev {
+		t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 0.0 != ps[0] {
+		t.Errorf("median: 0.0 != %v\n", ps[0])
+	}
+	if 0.0 != ps[1] {
+		t.Errorf("75th percentile: 0.0 != %v\n", ps[1])
+	}
+	if 0.0 != ps[2] {
+		t.Errorf("99th percentile: 0.0 != %v\n", ps[2])
+	}
+}
+
+func TestHistogramSnapshot(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	snapshot := h.Snapshot()
+	h.Update(0)
+	testHistogram10000(t, snapshot)
+}
+
+func testHistogram10000(t *testing.T, h Histogram) {
+	if count := h.Count(); 10000 != count {
+		t.Errorf("h.Count(): 10000 != %v\n", count)
+	}
+	if min := h.Min(); 1 != min {
+		t.Errorf("h.Min(): 1 != %v\n", min)
+	}
+	if max := h.Max(); 10000 != max {
+		t.Errorf("h.Max(): 10000 != %v\n", max)
+	}
+	if mean := h.Mean(); 5000.5 != mean {
+		t.Errorf("h.Mean(): 5000.5 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 2886.751331514372 != stdDev {
+		t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 5000.5 != ps[0] {
+		t.Errorf("median: 5000.5 != %v\n", ps[0])
+	}
+	if 7500.75 != ps[1] {
+		t.Errorf("75th percentile: 7500.75 != %v\n", ps[1])
+	}
+	if 9900.99 != ps[2] {
+		t.Errorf("99th percentile: 9900.99 != %v\n", ps[2])
+	}
+}
diff --git a/metrics/influxdb/LICENSE b/metrics/influxdb/LICENSE
new file mode 100644
index 0000000000..e5bf20cdb5
--- /dev/null
+++ b/metrics/influxdb/LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2015 Vincent Rischmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/metrics/influxdb/README.md b/metrics/influxdb/README.md
new file mode 100644
index 0000000000..b76b1a3f99
--- /dev/null
+++ b/metrics/influxdb/README.md
@@ -0,0 +1,30 @@
+go-metrics-influxdb
+===================
+
+This is a reporter for the [go-metrics](https://github.com/rcrowley/go-metrics) library which will post the metrics to [InfluxDB](https://influxdb.com/).
+
+Note
+----
+
+This is only compatible with InfluxDB 0.9+.
+
+Usage
+-----
+
+```go
+import "github.com/vrischmann/go-metrics-influxdb"
+
+go influxdb.InfluxDB(
+    metrics.DefaultRegistry, // metrics registry
+    time.Second * 10,        // interval
+    "http://localhost:8086", // the InfluxDB url
+    "mydb",                  // your InfluxDB database
+    "myuser",                // your InfluxDB user
+    "mypassword",            // your InfluxDB password
+)
+```
+
+License
+-------
+
+go-metrics-influxdb is licensed under the MIT license. See the LICENSE file for details.
diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go
new file mode 100644
index 0000000000..d5cb4da66f
--- /dev/null
+++ b/metrics/influxdb/influxdb.go
@@ -0,0 +1,227 @@
+package influxdb
+
+import (
+	"fmt"
+	"log"
+	uurl "net/url"
+	"time"
+
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/influxdata/influxdb/client"
+)
+
+type reporter struct {
+	reg      metrics.Registry
+	interval time.Duration
+
+	url       uurl.URL
+	database  string
+	username  string
+	password  string
+	namespace string
+	tags      map[string]string
+
+	client *client.Client
+
+	cache map[string]int64
+}
+
+// InfluxDB starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval.
+func InfluxDB(r metrics.Registry, d time.Duration, url, database, username, password, namespace string) {
+	InfluxDBWithTags(r, d, url, database, username, password, namespace, nil)
+}
+
+// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags
+func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, username, password, namespace string, tags map[string]string) {
+	u, err := uurl.Parse(url)
+	if err != nil {
+		log.Printf("unable to parse InfluxDB url %s. err=%v", url, err)
+		return
+	}
+
+	rep := &reporter{
+		reg:       r,
+		interval:  d,
+		url:       *u,
+		database:  database,
+		username:  username,
+		password:  password,
+		namespace: namespace,
+		tags:      tags,
+		cache:     make(map[string]int64),
+	}
+	if err := rep.makeClient(); err != nil {
+		log.Printf("unable to make InfluxDB client. err=%v", err)
+		return
+	}
+
+	rep.run()
+}
+
+func (r *reporter) makeClient() (err error) {
+	r.client, err = client.NewClient(client.Config{
+		URL:      r.url,
+		Username: r.username,
+		Password: r.password,
+	})
+
+	return
+}
+
+func (r *reporter) run() {
+	intervalTicker := time.Tick(r.interval)
+	pingTicker := time.Tick(time.Second * 5)
+
+	for {
+		select {
+		case <-intervalTicker:
+			if err := r.send(); err != nil {
+				log.Printf("unable to send to InfluxDB. err=%v", err)
+			}
+		case <-pingTicker:
+			_, _, err := r.client.Ping()
+			if err != nil {
+				log.Printf("got error while sending a ping to InfluxDB, trying to recreate client. err=%v", err)
+
+				if err = r.makeClient(); err != nil {
+					log.Printf("unable to make InfluxDB client. err=%v", err)
+				}
+			}
+		}
+	}
+}
+
+func (r *reporter) send() error {
+	var pts []client.Point
+
+	r.reg.Each(func(name string, i interface{}) {
+		now := time.Now()
+		namespace := r.namespace
+
+		switch metric := i.(type) {
+		case metrics.Counter:
+			v := metric.Count()
+			l := r.cache[name]
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.count", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": v - l,
+				},
+				Time: now,
+			})
+			r.cache[name] = v
+		case metrics.Gauge:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.gauge", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": ms.Value(),
+				},
+				Time: now,
+			})
+		case metrics.GaugeFloat64:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.gauge", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": ms.Value(),
+				},
+				Time: now,
+			})
+		case metrics.Histogram:
+			ms := metric.Snapshot()
+			ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999})
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.histogram", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count":    ms.Count(),
+					"max":      ms.Max(),
+					"mean":     ms.Mean(),
+					"min":      ms.Min(),
+					"stddev":   ms.StdDev(),
+					"variance": ms.Variance(),
+					"p50":      ps[0],
+					"p75":      ps[1],
+					"p95":      ps[2],
+					"p99":      ps[3],
+					"p999":     ps[4],
+					"p9999":    ps[5],
+				},
+				Time: now,
+			})
+		case metrics.Meter:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.meter", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count": ms.Count(),
+					"m1":    ms.Rate1(),
+					"m5":    ms.Rate5(),
+					"m15":   ms.Rate15(),
+					"mean":  ms.RateMean(),
+				},
+				Time: now,
+			})
+		case metrics.Timer:
+			ms := metric.Snapshot()
+			ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999})
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.timer", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count":    ms.Count(),
+					"max":      ms.Max(),
+					"mean":     ms.Mean(),
+					"min":      ms.Min(),
+					"stddev":   ms.StdDev(),
+					"variance": ms.Variance(),
+					"p50":      ps[0],
+					"p75":      ps[1],
+					"p95":      ps[2],
+					"p99":      ps[3],
+					"p999":     ps[4],
+					"p9999":    ps[5],
+					"m1":       ms.Rate1(),
+					"m5":       ms.Rate5(),
+					"m15":      ms.Rate15(),
+					"meanrate": ms.RateMean(),
+				},
+				Time: now,
+			})
+		case metrics.ResettingTimer:
+			t := metric.Snapshot()
+
+			if len(t.Values()) > 0 {
+				ps := t.Percentiles([]float64{50, 95, 99})
+				val := t.Values()
+				pts = append(pts, client.Point{
+					Measurement: fmt.Sprintf("%s%s.span", namespace, name),
+					Tags:        r.tags,
+					Fields: map[string]interface{}{
+						"count": len(val),
+						"max":   val[len(val)-1],
+						"mean":  t.Mean(),
+						"min":   val[0],
+						"p50":   ps[0],
+						"p95":   ps[1],
+						"p99":   ps[2],
+					},
+					Time: now,
+				})
+			}
+		}
+	})
+
+	bps := client.BatchPoints{
+		Points:   pts,
+		Database: r.database,
+	}
+
+	_, err := r.client.Write(bps)
+	return err
+}
diff --git a/metrics/init_test.go b/metrics/init_test.go
new file mode 100644
index 0000000000..43401e833c
--- /dev/null
+++ b/metrics/init_test.go
@@ -0,0 +1,5 @@
+package metrics
+
+func init() {
+	Enabled = true
+}
diff --git a/metrics/json.go b/metrics/json.go
new file mode 100644
index 0000000000..2087d8211e
--- /dev/null
+++ b/metrics/json.go
@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"encoding/json"
+	"io"
+	"time"
+)
+
+// MarshalJSON returns a byte slice containing a JSON representation of all
+// the metrics in the Registry.
+func (r *StandardRegistry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(r.GetAll())
+}
+
+// WriteJSON writes metrics from the given registry  periodically to the
+// specified io.Writer as JSON.
+func WriteJSON(r Registry, d time.Duration, w io.Writer) {
+	for range time.Tick(d) {
+		WriteJSONOnce(r, w)
+	}
+}
+
+// WriteJSONOnce writes metrics from the given registry to the specified
+// io.Writer as JSON.
+func WriteJSONOnce(r Registry, w io.Writer) {
+	json.NewEncoder(w).Encode(r)
+}
+
+func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.GetAll())
+}
diff --git a/metrics/json_test.go b/metrics/json_test.go
new file mode 100644
index 0000000000..cf70051f7a
--- /dev/null
+++ b/metrics/json_test.go
@@ -0,0 +1,28 @@
+package metrics
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+func TestRegistryMarshallJSON(t *testing.T) {
+	b := &bytes.Buffer{}
+	enc := json.NewEncoder(b)
+	r := NewRegistry()
+	r.Register("counter", NewCounter())
+	enc.Encode(r)
+	if s := b.String(); "{\"counter\":{\"count\":0}}\n" != s {
+		t.Fatalf(s)
+	}
+}
+
+func TestRegistryWriteJSONOnce(t *testing.T) {
+	r := NewRegistry()
+	r.Register("counter", NewCounter())
+	b := &bytes.Buffer{}
+	WriteJSONOnce(r, b)
+	if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" {
+		t.Fail()
+	}
+}
diff --git a/metrics/librato/client.go b/metrics/librato/client.go
new file mode 100644
index 0000000000..8c0c850e38
--- /dev/null
+++ b/metrics/librato/client.go
@@ -0,0 +1,102 @@
+package librato
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+const Operations = "operations"
+const OperationsShort = "ops"
+
+type LibratoClient struct {
+	Email, Token string
+}
+
+// property strings
+const (
+	// display attributes
+	Color             = "color"
+	DisplayMax        = "display_max"
+	DisplayMin        = "display_min"
+	DisplayUnitsLong  = "display_units_long"
+	DisplayUnitsShort = "display_units_short"
+	DisplayStacked    = "display_stacked"
+	DisplayTransform  = "display_transform"
+	// special gauge display attributes
+	SummarizeFunction = "summarize_function"
+	Aggregate         = "aggregate"
+
+	// metric keys
+	Name        = "name"
+	Period      = "period"
+	Description = "description"
+	DisplayName = "display_name"
+	Attributes  = "attributes"
+
+	// measurement keys
+	MeasureTime = "measure_time"
+	Source      = "source"
+	Value       = "value"
+
+	// special gauge keys
+	Count      = "count"
+	Sum        = "sum"
+	Max        = "max"
+	Min        = "min"
+	SumSquares = "sum_squares"
+
+	// batch keys
+	Counters = "counters"
+	Gauges   = "gauges"
+
+	MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics"
+)
+
+type Measurement map[string]interface{}
+type Metric map[string]interface{}
+
+type Batch struct {
+	Gauges      []Measurement `json:"gauges,omitempty"`
+	Counters    []Measurement `json:"counters,omitempty"`
+	MeasureTime int64         `json:"measure_time"`
+	Source      string        `json:"source"`
+}
+
+func (self *LibratoClient) PostMetrics(batch Batch) (err error) {
+	var (
+		js   []byte
+		req  *http.Request
+		resp *http.Response
+	)
+
+	if len(batch.Counters) == 0 && len(batch.Gauges) == 0 {
+		return nil
+	}
+
+	if js, err = json.Marshal(batch); err != nil {
+		return
+	}
+
+	if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil {
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(self.Email, self.Token)
+
+	if resp, err = http.DefaultClient.Do(req); err != nil {
+		return
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		var body []byte
+		if body, err = ioutil.ReadAll(resp.Body); err != nil {
+			body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err))
+		}
+		err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body))
+	}
+	return
+}
diff --git a/metrics/librato/librato.go b/metrics/librato/librato.go
new file mode 100644
index 0000000000..f8c8c9ecb7
--- /dev/null
+++ b/metrics/librato/librato.go
@@ -0,0 +1,235 @@
+package librato
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"regexp"
+	"time"
+
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+// a regexp for extracting the unit from time.Duration.String
+var unitRegexp = regexp.MustCompile(`[^\\d]+$`)
+
+// a helper that turns a time.Duration into librato display attributes for timer metrics
+func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) {
+	attrs = make(map[string]interface{})
+	attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d))
+	attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String())))
+	return
+}
+
+type Reporter struct {
+	Email, Token    string
+	Namespace       string
+	Source          string
+	Interval        time.Duration
+	Registry        metrics.Registry
+	Percentiles     []float64              // percentiles to report on histogram metrics
+	TimerAttributes map[string]interface{} // units in which timers will be displayed
+	intervalSec     int64
+}
+
+func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter {
+	return &Reporter{e, t, "", s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
+}
+
+func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) {
+	NewReporter(r, d, e, t, s, p, u).Run()
+}
+
+func (self *Reporter) Run() {
+	log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015")
+	ticker := time.Tick(self.Interval)
+	metricsApi := &LibratoClient{self.Email, self.Token}
+	for now := range ticker {
+		var metrics Batch
+		var err error
+		if metrics, err = self.BuildRequest(now, self.Registry); err != nil {
+			log.Printf("ERROR constructing librato request body %s", err)
+			continue
+		}
+		if err := metricsApi.PostMetrics(metrics); err != nil {
+			log.Printf("ERROR sending metrics to librato %s", err)
+			continue
+		}
+	}
+}
+
+// calculate sum of squares from data provided by metrics.Histogram
+// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
+func sumSquares(s metrics.Sample) float64 {
+	count := float64(s.Count())
+	sumSquared := math.Pow(count*s.Mean(), 2)
+	sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+func sumSquaresTimer(t metrics.Timer) float64 {
+	count := float64(t.Count())
+	sumSquared := math.Pow(count*t.Mean(), 2)
+	sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+
+func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) {
+	snapshot = Batch{
+		// coerce timestamps to a stepping fn so that they line up in Librato graphs
+		MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec,
+		Source:      self.Source,
+	}
+	snapshot.Gauges = make([]Measurement, 0)
+	snapshot.Counters = make([]Measurement, 0)
+	histogramGaugeCount := 1 + len(self.Percentiles)
+	r.Each(func(name string, metric interface{}) {
+		if self.Namespace != "" {
+			name = fmt.Sprintf("%s.%s", self.Namespace, name)
+		}
+		measurement := Measurement{}
+		measurement[Period] = self.Interval.Seconds()
+		switch m := metric.(type) {
+		case metrics.Counter:
+			if m.Count() > 0 {
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "count")
+				measurement[Value] = float64(m.Count())
+				measurement[Attributes] = map[string]interface{}{
+					DisplayUnitsLong:  Operations,
+					DisplayUnitsShort: OperationsShort,
+					DisplayMin:        "0",
+				}
+				snapshot.Counters = append(snapshot.Counters, measurement)
+			}
+		case metrics.Gauge:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Value())
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.GaugeFloat64:
+			measurement[Name] = name
+			measurement[Value] = m.Value()
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.Histogram:
+			if m.Count() > 0 {
+				gauges := make([]Measurement, histogramGaugeCount)
+				s := m.Sample()
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "hist")
+				measurement[Count] = uint64(s.Count())
+				measurement[Max] = float64(s.Max())
+				measurement[Min] = float64(s.Min())
+				measurement[Sum] = float64(s.Sum())
+				measurement[SumSquares] = sumSquares(s)
+				gauges[0] = measurement
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:   fmt.Sprintf("%s.%.2f", measurement[Name], p),
+						Value:  s.Percentile(p),
+						Period: measurement[Period],
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+			}
+		case metrics.Meter:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			snapshot.Gauges = append(snapshot.Gauges,
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "1min"),
+					Value:  m.Rate1(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "5min"),
+					Value:  m.Rate5(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "15min"),
+					Value:  m.Rate15(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+			)
+		case metrics.Timer:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			if m.Count() > 0 {
+				libratoName := fmt.Sprintf("%s.%s", name, "timer.mean")
+				gauges := make([]Measurement, histogramGaugeCount)
+				gauges[0] = Measurement{
+					Name:       libratoName,
+					Count:      uint64(m.Count()),
+					Sum:        m.Mean() * float64(m.Count()),
+					Max:        float64(m.Max()),
+					Min:        float64(m.Min()),
+					SumSquares: sumSquaresTimer(m),
+					Period:     int64(self.Interval.Seconds()),
+					Attributes: self.TimerAttributes,
+				}
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:       fmt.Sprintf("%s.timer.%2.0f", name, p*100),
+						Value:      m.Percentile(p),
+						Period:     int64(self.Interval.Seconds()),
+						Attributes: self.TimerAttributes,
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+				snapshot.Gauges = append(snapshot.Gauges,
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.1min"),
+						Value:  m.Rate1(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.5min"),
+						Value:  m.Rate5(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.15min"),
+						Value:  m.Rate15(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+				)
+			}
+		}
+	})
+	return
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/log.go b/metrics/log.go
similarity index 98%
rename from vendor/github.com/rcrowley/go-metrics/log.go
rename to metrics/log.go
index f8074c0457..0c8ea7c971 100644
--- a/vendor/github.com/rcrowley/go-metrics/log.go
+++ b/metrics/log.go
@@ -18,7 +18,7 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) {
 	du := float64(scale)
 	duSuffix := scale.String()[1:]
 
-	for _ = range time.Tick(freq) {
+	for range time.Tick(freq) {
 		r.Each(func(name string, i interface{}) {
 			switch metric := i.(type) {
 			case Counter:
diff --git a/vendor/github.com/rcrowley/go-metrics/memory.md b/metrics/memory.md
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/memory.md
rename to metrics/memory.md
diff --git a/vendor/github.com/rcrowley/go-metrics/meter.go b/metrics/meter.go
similarity index 81%
rename from vendor/github.com/rcrowley/go-metrics/meter.go
rename to metrics/meter.go
index 0389ab0b8f..82b2141a62 100644
--- a/vendor/github.com/rcrowley/go-metrics/meter.go
+++ b/metrics/meter.go
@@ -15,10 +15,13 @@ type Meter interface {
 	Rate15() float64
 	RateMean() float64
 	Snapshot() Meter
+	Stop()
 }
 
 // GetOrRegisterMeter returns an existing Meter or constructs and registers a
 // new StandardMeter.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func GetOrRegisterMeter(name string, r Registry) Meter {
 	if nil == r {
 		r = DefaultRegistry
@@ -27,14 +30,15 @@ func GetOrRegisterMeter(name string, r Registry) Meter {
 }
 
 // NewMeter constructs a new StandardMeter and launches a goroutine.
+// Be sure to call Stop() once the meter is of no use to allow for garbage collection.
 func NewMeter() Meter {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilMeter{}
 	}
 	m := newStandardMeter()
 	arbiter.Lock()
 	defer arbiter.Unlock()
-	arbiter.meters = append(arbiter.meters, m)
+	arbiter.meters[m] = struct{}{}
 	if !arbiter.started {
 		arbiter.started = true
 		go arbiter.tick()
@@ -44,6 +48,8 @@ func NewMeter() Meter {
 
 // NewMeter constructs and registers a new StandardMeter and launches a
 // goroutine.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func NewRegisteredMeter(name string, r Registry) Meter {
 	c := NewMeter()
 	if nil == r {
@@ -86,6 +92,9 @@ func (m *MeterSnapshot) RateMean() float64 { return m.rateMean }
 // Snapshot returns the snapshot.
 func (m *MeterSnapshot) Snapshot() Meter { return m }
 
+// Stop is a no-op.
+func (m *MeterSnapshot) Stop() {}
+
 // NilMeter is a no-op Meter.
 type NilMeter struct{}
 
@@ -110,12 +119,16 @@ func (NilMeter) RateMean() float64 { return 0.0 }
 // Snapshot is a no-op.
 func (NilMeter) Snapshot() Meter { return NilMeter{} }
 
+// Stop is a no-op.
+func (NilMeter) Stop() {}
+
 // StandardMeter is the standard implementation of a Meter.
 type StandardMeter struct {
 	lock        sync.RWMutex
 	snapshot    *MeterSnapshot
 	a1, a5, a15 EWMA
 	startTime   time.Time
+	stopped     bool
 }
 
 func newStandardMeter() *StandardMeter {
@@ -128,6 +141,19 @@ func newStandardMeter() *StandardMeter {
 	}
 }
 
+// Stop stops the meter, Mark() will be a no-op if you use it after being stopped.
+func (m *StandardMeter) Stop() {
+	m.lock.Lock()
+	stopped := m.stopped
+	m.stopped = true
+	m.lock.Unlock()
+	if !stopped {
+		arbiter.Lock()
+		delete(arbiter.meters, m)
+		arbiter.Unlock()
+	}
+}
+
 // Count returns the number of events recorded.
 func (m *StandardMeter) Count() int64 {
 	m.lock.RLock()
@@ -136,10 +162,13 @@ func (m *StandardMeter) Count() int64 {
 	return count
 }
 
-// Mark records the occurance of n events.
+// Mark records the occurrence of n events.
 func (m *StandardMeter) Mark(n int64) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
+	if m.stopped {
+		return
+	}
 	m.snapshot.count += n
 	m.a1.Update(n)
 	m.a5.Update(n)
@@ -205,29 +234,28 @@ func (m *StandardMeter) tick() {
 	m.updateSnapshot()
 }
 
+// meterArbiter ticks meters every 5s from a single goroutine.
+// meters are references in a set for future stopping.
 type meterArbiter struct {
 	sync.RWMutex
 	started bool
-	meters  []*StandardMeter
+	meters  map[*StandardMeter]struct{}
 	ticker  *time.Ticker
 }
 
-var arbiter = meterArbiter{ticker: time.NewTicker(5e9)}
+var arbiter = meterArbiter{ticker: time.NewTicker(5e9), meters: make(map[*StandardMeter]struct{})}
 
 // Ticks meters on the scheduled interval
 func (ma *meterArbiter) tick() {
-	for {
-		select {
-		case <-ma.ticker.C:
-			ma.tickMeters()
-		}
+	for range ma.ticker.C {
+		ma.tickMeters()
 	}
 }
 
 func (ma *meterArbiter) tickMeters() {
 	ma.RLock()
 	defer ma.RUnlock()
-	for _, meter := range ma.meters {
+	for meter := range ma.meters {
 		meter.tick()
 	}
 }
diff --git a/metrics/meter_test.go b/metrics/meter_test.go
new file mode 100644
index 0000000000..e889222601
--- /dev/null
+++ b/metrics/meter_test.go
@@ -0,0 +1,73 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+)
+
+func BenchmarkMeter(b *testing.B) {
+	m := NewMeter()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Mark(1)
+	}
+}
+
+func TestGetOrRegisterMeter(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredMeter("foo", r).Mark(47)
+	if m := GetOrRegisterMeter("foo", r); 47 != m.Count() {
+		t.Fatal(m)
+	}
+}
+
+func TestMeterDecay(t *testing.T) {
+	ma := meterArbiter{
+		ticker: time.NewTicker(time.Millisecond),
+		meters: make(map[*StandardMeter]struct{}),
+	}
+	m := newStandardMeter()
+	ma.meters[m] = struct{}{}
+	go ma.tick()
+	m.Mark(1)
+	rateMean := m.RateMean()
+	time.Sleep(100 * time.Millisecond)
+	if m.RateMean() >= rateMean {
+		t.Error("m.RateMean() didn't decrease")
+	}
+}
+
+func TestMeterNonzero(t *testing.T) {
+	m := NewMeter()
+	m.Mark(3)
+	if count := m.Count(); 3 != count {
+		t.Errorf("m.Count(): 3 != %v\n", count)
+	}
+}
+
+func TestMeterStop(t *testing.T) {
+	l := len(arbiter.meters)
+	m := NewMeter()
+	if len(arbiter.meters) != l+1 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters))
+	}
+	m.Stop()
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters))
+	}
+}
+
+func TestMeterSnapshot(t *testing.T) {
+	m := NewMeter()
+	m.Mark(1)
+	if snapshot := m.Snapshot(); m.RateMean() != snapshot.RateMean() {
+		t.Fatal(snapshot)
+	}
+}
+
+func TestMeterZero(t *testing.T) {
+	m := NewMeter()
+	if count := m.Count(); 0 != count {
+		t.Errorf("m.Count(): 0 != %v\n", count)
+	}
+}
diff --git a/metrics/metrics.go b/metrics/metrics.go
index c82661d802..e24324814c 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -1,20 +1,8 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
+// Go port of Coda Hale's Metrics library
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
+// 
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-// Package metrics provides general system and process level metrics collection.
+// Coda Hale's original work: 
 package metrics
 
 import (
@@ -24,17 +12,19 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/rcrowley/go-metrics"
-	"github.com/rcrowley/go-metrics/exp"
 )
 
+// Enabled is checked by the constructor functions for all of the
+// standard metrics.  If it is true, the metric returned is a stub.
+//
+// This global kill-switch helps quantify the observer effect and makes
+// for less cluttered pprof profiles.
+var Enabled bool = false
+
 // MetricsEnabledFlag is the CLI flag name to use to enable metrics collections.
 const MetricsEnabledFlag = "metrics"
 const DashboardEnabledFlag = "dashboard"
 
-// Enabled is the flag specifying if metrics are enable or not.
-var Enabled = false
-
 // Init enables or disables the metrics system. Since we need this to run before
 // any other code gets to create meters and timers, we'll actually do an ugly hack
 // and peek into the command line args for the metrics flag.
@@ -45,34 +35,6 @@ func init() {
 			Enabled = true
 		}
 	}
-	exp.Exp(metrics.DefaultRegistry)
-}
-
-// NewCounter create a new metrics Counter, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewCounter(name string) metrics.Counter {
-	if !Enabled {
-		return new(metrics.NilCounter)
-	}
-	return metrics.GetOrRegisterCounter(name, metrics.DefaultRegistry)
-}
-
-// NewMeter create a new metrics Meter, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewMeter(name string) metrics.Meter {
-	if !Enabled {
-		return new(metrics.NilMeter)
-	}
-	return metrics.GetOrRegisterMeter(name, metrics.DefaultRegistry)
-}
-
-// NewTimer create a new metrics Timer, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewTimer(name string) metrics.Timer {
-	if !Enabled {
-		return new(metrics.NilTimer)
-	}
-	return metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry)
 }
 
 // CollectProcessMetrics periodically collects various metrics about the running
@@ -90,17 +52,17 @@ func CollectProcessMetrics(refresh time.Duration) {
 		diskstats[i] = new(DiskStats)
 	}
 	// Define the various metrics to collect
-	memAllocs := metrics.GetOrRegisterMeter("system/memory/allocs", metrics.DefaultRegistry)
-	memFrees := metrics.GetOrRegisterMeter("system/memory/frees", metrics.DefaultRegistry)
-	memInuse := metrics.GetOrRegisterMeter("system/memory/inuse", metrics.DefaultRegistry)
-	memPauses := metrics.GetOrRegisterMeter("system/memory/pauses", metrics.DefaultRegistry)
+	memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
+	memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
+	memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry)
+	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
 
-	var diskReads, diskReadBytes, diskWrites, diskWriteBytes metrics.Meter
+	var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
 	if err := ReadDiskStats(diskstats[0]); err == nil {
-		diskReads = metrics.GetOrRegisterMeter("system/disk/readcount", metrics.DefaultRegistry)
-		diskReadBytes = metrics.GetOrRegisterMeter("system/disk/readdata", metrics.DefaultRegistry)
-		diskWrites = metrics.GetOrRegisterMeter("system/disk/writecount", metrics.DefaultRegistry)
-		diskWriteBytes = metrics.GetOrRegisterMeter("system/disk/writedata", metrics.DefaultRegistry)
+		diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
+		diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
+		diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
+		diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
 	} else {
 		log.Debug("Failed to read disk metrics", "err", err)
 	}
diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go
new file mode 100644
index 0000000000..726fba3475
--- /dev/null
+++ b/metrics/metrics_test.go
@@ -0,0 +1,124 @@
+package metrics
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"sync"
+	"testing"
+)
+
+const FANOUT = 128
+
+// Stop the compiler from complaining during debugging.
+var (
+	_ = ioutil.Discard
+	_ = log.LstdFlags
+)
+
+func BenchmarkMetrics(b *testing.B) {
+	r := NewRegistry()
+	c := NewRegisteredCounter("counter", r)
+	g := NewRegisteredGauge("gauge", r)
+	gf := NewRegisteredGaugeFloat64("gaugefloat64", r)
+	h := NewRegisteredHistogram("histogram", r, NewUniformSample(100))
+	m := NewRegisteredMeter("meter", r)
+	t := NewRegisteredTimer("timer", r)
+	RegisterDebugGCStats(r)
+	RegisterRuntimeMemStats(r)
+	b.ResetTimer()
+	ch := make(chan bool)
+
+	wgD := &sync.WaitGroup{}
+	/*
+		wgD.Add(1)
+		go func() {
+			defer wgD.Done()
+			//log.Println("go CaptureDebugGCStats")
+			for {
+				select {
+				case <-ch:
+					//log.Println("done CaptureDebugGCStats")
+					return
+				default:
+					CaptureDebugGCStatsOnce(r)
+				}
+			}
+		}()
+	//*/
+
+	wgR := &sync.WaitGroup{}
+	//*
+	wgR.Add(1)
+	go func() {
+		defer wgR.Done()
+		//log.Println("go CaptureRuntimeMemStats")
+		for {
+			select {
+			case <-ch:
+				//log.Println("done CaptureRuntimeMemStats")
+				return
+			default:
+				CaptureRuntimeMemStatsOnce(r)
+			}
+		}
+	}()
+	//*/
+
+	wgW := &sync.WaitGroup{}
+	/*
+		wgW.Add(1)
+		go func() {
+			defer wgW.Done()
+			//log.Println("go Write")
+			for {
+				select {
+				case <-ch:
+					//log.Println("done Write")
+					return
+				default:
+					WriteOnce(r, ioutil.Discard)
+				}
+			}
+		}()
+	//*/
+
+	wg := &sync.WaitGroup{}
+	wg.Add(FANOUT)
+	for i := 0; i < FANOUT; i++ {
+		go func(i int) {
+			defer wg.Done()
+			//log.Println("go", i)
+			for i := 0; i < b.N; i++ {
+				c.Inc(1)
+				g.Update(int64(i))
+				gf.Update(float64(i))
+				h.Update(int64(i))
+				m.Mark(1)
+				t.Update(1)
+			}
+			//log.Println("done", i)
+		}(i)
+	}
+	wg.Wait()
+	close(ch)
+	wgD.Wait()
+	wgR.Wait()
+	wgW.Wait()
+}
+
+func Example() {
+	c := NewCounter()
+	Register("money", c)
+	c.Inc(17)
+
+	// Threadsafe registration
+	t := GetOrRegisterTimer("db.get.latency", nil)
+	t.Time(func() {})
+	t.Update(1)
+
+	fmt.Println(c.Count())
+	fmt.Println(t.Min())
+	// Output: 17
+	// 1
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/opentsdb.go b/metrics/opentsdb.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/opentsdb.go
rename to metrics/opentsdb.go
index 266b6c93d2..df7f152ed2 100644
--- a/vendor/github.com/rcrowley/go-metrics/opentsdb.go
+++ b/metrics/opentsdb.go
@@ -38,7 +38,7 @@ func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
 // OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB,
 // but it takes a OpenTSDBConfig instead.
 func OpenTSDBWithConfig(c OpenTSDBConfig) {
-	for _ = range time.Tick(c.FlushInterval) {
+	for range time.Tick(c.FlushInterval) {
 		if err := openTSDB(&c); nil != err {
 			log.Println(err)
 		}
diff --git a/metrics/opentsdb_test.go b/metrics/opentsdb_test.go
new file mode 100644
index 0000000000..c43728960e
--- /dev/null
+++ b/metrics/opentsdb_test.go
@@ -0,0 +1,21 @@
+package metrics
+
+import (
+	"net"
+	"time"
+)
+
+func ExampleOpenTSDB() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr)
+}
+
+func ExampleOpenTSDBWithConfig() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go OpenTSDBWithConfig(OpenTSDBConfig{
+		Addr:          addr,
+		Registry:      DefaultRegistry,
+		FlushInterval: 1 * time.Second,
+		DurationUnit:  time.Millisecond,
+	})
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/registry.go b/metrics/registry.go
similarity index 76%
rename from vendor/github.com/rcrowley/go-metrics/registry.go
rename to metrics/registry.go
index 2bb7a1e7d0..cc34c9dfd2 100644
--- a/vendor/github.com/rcrowley/go-metrics/registry.go
+++ b/metrics/registry.go
@@ -29,6 +29,9 @@ type Registry interface {
 	// Get the metric by the given name or nil if none is registered.
 	Get(string) interface{}
 
+	// GetAll metrics in the Registry.
+	GetAll() map[string]map[string]interface{}
+
 	// Gets an existing metric or registers the given one.
 	// The interface can be the metric to register if not found in registry,
 	// or a function returning the metric for lazy instantiation.
@@ -109,10 +112,72 @@ func (r *StandardRegistry) RunHealthchecks() {
 	}
 }
 
+// GetAll metrics in the Registry
+func (r *StandardRegistry) GetAll() map[string]map[string]interface{} {
+	data := make(map[string]map[string]interface{})
+	r.Each(func(name string, i interface{}) {
+		values := make(map[string]interface{})
+		switch metric := i.(type) {
+		case Counter:
+			values["count"] = metric.Count()
+		case Gauge:
+			values["value"] = metric.Value()
+		case GaugeFloat64:
+			values["value"] = metric.Value()
+		case Healthcheck:
+			values["error"] = nil
+			metric.Check()
+			if err := metric.Error(); nil != err {
+				values["error"] = metric.Error().Error()
+			}
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = h.Count()
+			values["min"] = h.Min()
+			values["max"] = h.Max()
+			values["mean"] = h.Mean()
+			values["stddev"] = h.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+		case Meter:
+			m := metric.Snapshot()
+			values["count"] = m.Count()
+			values["1m.rate"] = m.Rate1()
+			values["5m.rate"] = m.Rate5()
+			values["15m.rate"] = m.Rate15()
+			values["mean.rate"] = m.RateMean()
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = t.Count()
+			values["min"] = t.Min()
+			values["max"] = t.Max()
+			values["mean"] = t.Mean()
+			values["stddev"] = t.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+			values["1m.rate"] = t.Rate1()
+			values["5m.rate"] = t.Rate5()
+			values["15m.rate"] = t.Rate15()
+			values["mean.rate"] = t.RateMean()
+		}
+		data[name] = values
+	})
+	return data
+}
+
 // Unregister the metric with the given name.
 func (r *StandardRegistry) Unregister(name string) {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
+	r.stop(name)
 	delete(r.metrics, name)
 }
 
@@ -120,7 +185,8 @@ func (r *StandardRegistry) Unregister(name string) {
 func (r *StandardRegistry) UnregisterAll() {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
-	for name, _ := range r.metrics {
+	for name := range r.metrics {
+		r.stop(name)
 		delete(r.metrics, name)
 	}
 }
@@ -130,7 +196,7 @@ func (r *StandardRegistry) register(name string, i interface{}) error {
 		return DuplicateMetric(name)
 	}
 	switch i.(type) {
-	case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer:
+	case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer, ResettingTimer:
 		r.metrics[name] = i
 	}
 	return nil
@@ -146,6 +212,19 @@ func (r *StandardRegistry) registered() map[string]interface{} {
 	return metrics
 }
 
+func (r *StandardRegistry) stop(name string) {
+	if i, ok := r.metrics[name]; ok {
+		if s, ok := i.(Stoppable); ok {
+			s.Stop()
+		}
+	}
+}
+
+// Stoppable defines the metrics which has to be stopped.
+type Stoppable interface {
+	Stop()
+}
+
 type PrefixedRegistry struct {
 	underlying Registry
 	prefix     string
@@ -216,6 +295,11 @@ func (r *PrefixedRegistry) RunHealthchecks() {
 	r.underlying.RunHealthchecks()
 }
 
+// GetAll metrics in the Registry
+func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} {
+	return r.underlying.GetAll()
+}
+
 // Unregister the metric with the given name. The name will be prefixed.
 func (r *PrefixedRegistry) Unregister(name string) {
 	realName := r.prefix + name
diff --git a/metrics/registry_test.go b/metrics/registry_test.go
new file mode 100644
index 0000000000..a63e485fe9
--- /dev/null
+++ b/metrics/registry_test.go
@@ -0,0 +1,305 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func BenchmarkRegistry(b *testing.B) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		r.Each(func(string, interface{}) {})
+	}
+}
+
+func TestRegistry(t *testing.T) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if "foo" != name {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if 1 != i {
+		t.Fatal(i)
+	}
+	r.Unregister("foo")
+	i = 0
+	r.Each(func(string, interface{}) { i++ })
+	if 0 != i {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryDuplicate(t *testing.T) {
+	r := NewRegistry()
+	if err := r.Register("foo", NewCounter()); nil != err {
+		t.Fatal(err)
+	}
+	if err := r.Register("foo", NewGauge()); nil == err {
+		t.Fatal(err)
+	}
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if 1 != i {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryGet(t *testing.T) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	if count := r.Get("foo").(Counter).Count(); 0 != count {
+		t.Fatal(count)
+	}
+	r.Get("foo").(Counter).Inc(1)
+	if count := r.Get("foo").(Counter).Count(); 1 != count {
+		t.Fatal(count)
+	}
+}
+
+func TestRegistryGetOrRegister(t *testing.T) {
+	r := NewRegistry()
+
+	// First metric wins with GetOrRegister
+	_ = r.GetOrRegister("foo", NewCounter())
+	m := r.GetOrRegister("foo", NewGauge())
+	if _, ok := m.(Counter); !ok {
+		t.Fatal(m)
+	}
+
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if name != "foo" {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) {
+	r := NewRegistry()
+
+	// First metric wins with GetOrRegister
+	_ = r.GetOrRegister("foo", NewCounter)
+	m := r.GetOrRegister("foo", NewGauge)
+	if _, ok := m.(Counter); !ok {
+		t.Fatal(m)
+	}
+
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if name != "foo" {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryUnregister(t *testing.T) {
+	l := len(arbiter.meters)
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	r.Register("bar", NewMeter())
+	r.Register("baz", NewTimer())
+	if len(arbiter.meters) != l+2 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters))
+	}
+	r.Unregister("foo")
+	r.Unregister("bar")
+	r.Unregister("baz")
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters))
+	}
+}
+
+func TestPrefixedChildRegistryGetOrRegister(t *testing.T) {
+	r := NewRegistry()
+	pr := NewPrefixedChildRegistry(r, "prefix.")
+
+	_ = pr.GetOrRegister("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryGetOrRegister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+
+	_ = r.GetOrRegister("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryRegister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+	err := r.Register("foo", NewCounter())
+	c := NewCounter()
+	Register("bar", c)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryUnregister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+
+	_ = r.Register("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+
+	r.Unregister("foo")
+
+	i = 0
+	r.Each(func(name string, m interface{}) {
+		i++
+	})
+
+	if i != 0 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryGet(t *testing.T) {
+	pr := NewPrefixedRegistry("prefix.")
+	name := "foo"
+	pr.Register(name, NewCounter())
+
+	fooCounter := pr.Get(name)
+	if fooCounter == nil {
+		t.Fatal(name)
+	}
+}
+
+func TestPrefixedChildRegistryGet(t *testing.T) {
+	r := NewRegistry()
+	pr := NewPrefixedChildRegistry(r, "prefix.")
+	name := "foo"
+	pr.Register(name, NewCounter())
+	fooCounter := pr.Get(name)
+	if fooCounter == nil {
+		t.Fatal(name)
+	}
+}
+
+func TestChildPrefixedRegistryRegister(t *testing.T) {
+	r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.")
+	err := r.Register("foo", NewCounter())
+	c := NewCounter()
+	Register("bar", c)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestChildPrefixedRegistryOfChildRegister(t *testing.T) {
+	r := NewPrefixedChildRegistry(NewRegistry(), "prefix.")
+	r2 := NewPrefixedChildRegistry(r, "prefix2.")
+	err := r.Register("foo2", NewCounter())
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = r2.Register("baz", NewCounter())
+	c := NewCounter()
+	Register("bars", c)
+
+	i := 0
+	r2.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.prefix2.baz" {
+			//t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestWalkRegistries(t *testing.T) {
+	r := NewPrefixedChildRegistry(NewRegistry(), "prefix.")
+	r2 := NewPrefixedChildRegistry(r, "prefix2.")
+	err := r.Register("foo2", NewCounter())
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = r2.Register("baz", NewCounter())
+	c := NewCounter()
+	Register("bars", c)
+
+	_, prefix := findPrefix(r2, "")
+	if "prefix.prefix2." != prefix {
+		t.Fatal(prefix)
+	}
+
+}
diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go
new file mode 100644
index 0000000000..57bcb31343
--- /dev/null
+++ b/metrics/resetting_timer.go
@@ -0,0 +1,237 @@
+package metrics
+
+import (
+	"math"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Initial slice capacity for the values stored in a ResettingTimer
+const InitialResettingTimerSliceCap = 10
+
+// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval.
+type ResettingTimer interface {
+	Values() []int64
+	Snapshot() ResettingTimer
+	Percentiles([]float64) []int64
+	Mean() float64
+	Time(func())
+	Update(time.Duration)
+	UpdateSince(time.Time)
+}
+
+// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a
+// new StandardResettingTimer.
+func GetOrRegisterResettingTimer(name string, r Registry) ResettingTimer {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewResettingTimer).(ResettingTimer)
+}
+
+// NewRegisteredResettingTimer constructs and registers a new StandardResettingTimer.
+func NewRegisteredResettingTimer(name string, r Registry) ResettingTimer {
+	c := NewResettingTimer()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// NewResettingTimer constructs a new StandardResettingTimer
+func NewResettingTimer() ResettingTimer {
+	if !Enabled {
+		return NilResettingTimer{}
+	}
+	return &StandardResettingTimer{
+		values: make([]int64, 0, InitialResettingTimerSliceCap),
+	}
+}
+
+// NilResettingTimer is a no-op ResettingTimer.
+type NilResettingTimer struct {
+}
+
+// Values is a no-op.
+func (NilResettingTimer) Values() []int64 { return nil }
+
+// Snapshot is a no-op.
+func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} }
+
+// Time is a no-op.
+func (NilResettingTimer) Time(func()) {}
+
+// Update is a no-op.
+func (NilResettingTimer) Update(time.Duration) {}
+
+// Percentiles panics.
+func (NilResettingTimer) Percentiles([]float64) []int64 {
+	panic("Percentiles called on a NilResettingTimer")
+}
+
+// Mean panics.
+func (NilResettingTimer) Mean() float64 {
+	panic("Mean called on a NilResettingTimer")
+}
+
+// UpdateSince is a no-op.
+func (NilResettingTimer) UpdateSince(time.Time) {}
+
+// StandardResettingTimer is the standard implementation of a ResettingTimer.
+// and Meter.
+type StandardResettingTimer struct {
+	values []int64
+	mutex  sync.Mutex
+}
+
+// Values returns a slice with all measurements.
+func (t *StandardResettingTimer) Values() []int64 {
+	return t.values
+}
+
+// Snapshot resets the timer and returns a read-only copy of its contents.
+func (t *StandardResettingTimer) Snapshot() ResettingTimer {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	currentValues := t.values
+	t.values = make([]int64, 0, InitialResettingTimerSliceCap)
+
+	return &ResettingTimerSnapshot{
+		values: currentValues,
+	}
+}
+
+// Percentiles panics.
+func (t *StandardResettingTimer) Percentiles([]float64) []int64 {
+	panic("Percentiles called on a StandardResettingTimer")
+}
+
+// Mean panics.
+func (t *StandardResettingTimer) Mean() float64 {
+	panic("Mean called on a StandardResettingTimer")
+}
+
+// Record the duration of the execution of the given function.
+func (t *StandardResettingTimer) Time(f func()) {
+	ts := time.Now()
+	f()
+	t.Update(time.Since(ts))
+}
+
+// Record the duration of an event.
+func (t *StandardResettingTimer) Update(d time.Duration) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.values = append(t.values, int64(d))
+}
+
+// Record the duration of an event that started at a time and ends now.
+func (t *StandardResettingTimer) UpdateSince(ts time.Time) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.values = append(t.values, int64(time.Since(ts)))
+}
+
+// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer.
+type ResettingTimerSnapshot struct {
+	values              []int64
+	mean                float64
+	thresholdBoundaries []int64
+	calculated          bool
+}
+
+// Snapshot returns the snapshot.
+func (t *ResettingTimerSnapshot) Snapshot() ResettingTimer { return t }
+
+// Time panics.
+func (*ResettingTimerSnapshot) Time(func()) {
+	panic("Time called on a ResettingTimerSnapshot")
+}
+
+// Update panics.
+func (*ResettingTimerSnapshot) Update(time.Duration) {
+	panic("Update called on a ResettingTimerSnapshot")
+}
+
+// UpdateSince panics.
+func (*ResettingTimerSnapshot) UpdateSince(time.Time) {
+	panic("UpdateSince called on a ResettingTimerSnapshot")
+}
+
+// Values returns all values from snapshot.
+func (t *ResettingTimerSnapshot) Values() []int64 {
+	return t.values
+}
+
+// Percentiles returns the boundaries for the input percentiles.
+func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []int64 {
+	t.calc(percentiles)
+
+	return t.thresholdBoundaries
+}
+
+// Mean returns the mean of the snapshotted values
+func (t *ResettingTimerSnapshot) Mean() float64 {
+	if !t.calculated {
+		t.calc([]float64{})
+	}
+
+	return t.mean
+}
+
+func (t *ResettingTimerSnapshot) calc(percentiles []float64) {
+	sort.Sort(Int64Slice(t.values))
+
+	count := len(t.values)
+	if count > 0 {
+		min := t.values[0]
+		max := t.values[count-1]
+
+		cumulativeValues := make([]int64, count)
+		cumulativeValues[0] = min
+		for i := 1; i < count; i++ {
+			cumulativeValues[i] = t.values[i] + cumulativeValues[i-1]
+		}
+
+		t.thresholdBoundaries = make([]int64, len(percentiles))
+
+		thresholdBoundary := max
+
+		for i, pct := range percentiles {
+			if count > 1 {
+				var abs float64
+				if pct >= 0 {
+					abs = pct
+				} else {
+					abs = 100 + pct
+				}
+				// poor man's math.Round(x):
+				// math.Floor(x + 0.5)
+				indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5))
+				if pct >= 0 {
+					indexOfPerc -= 1 // index offset=0
+				}
+				thresholdBoundary = t.values[indexOfPerc]
+			}
+
+			t.thresholdBoundaries[i] = thresholdBoundary
+		}
+
+		sum := cumulativeValues[count-1]
+		t.mean = float64(sum) / float64(count)
+	} else {
+		t.thresholdBoundaries = make([]int64, len(percentiles))
+		t.mean = 0
+	}
+
+	t.calculated = true
+}
+
+// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order.
+type Int64Slice []int64
+
+func (s Int64Slice) Len() int           { return len(s) }
+func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
+func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go
new file mode 100644
index 0000000000..58fd47f352
--- /dev/null
+++ b/metrics/resetting_timer_test.go
@@ -0,0 +1,106 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+)
+
+func TestResettingTimer(t *testing.T) {
+	tests := []struct {
+		values   []int64
+		start    int
+		end      int
+		wantP50  int64
+		wantP95  int64
+		wantP99  int64
+		wantMean float64
+		wantMin  int64
+		wantMax  int64
+	}{
+		{
+			values:  []int64{},
+			start:   1,
+			end:     11,
+			wantP50: 5, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+		{
+			values:  []int64{},
+			start:   1,
+			end:     101,
+			wantP50: 50, wantP95: 95, wantP99: 99,
+			wantMin: 1, wantMax: 100, wantMean: 50.5,
+		},
+		{
+			values:  []int64{1},
+			start:   0,
+			end:     0,
+			wantP50: 1, wantP95: 1, wantP99: 1,
+			wantMin: 1, wantMax: 1, wantMean: 1,
+		},
+		{
+			values:  []int64{0},
+			start:   0,
+			end:     0,
+			wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{},
+			start:   0,
+			end:     0,
+			wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{1, 10},
+			start:   0,
+			end:     0,
+			wantP50: 1, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+	}
+	for ind, tt := range tests {
+		timer := NewResettingTimer()
+
+		for i := tt.start; i < tt.end; i++ {
+			tt.values = append(tt.values, int64(i))
+		}
+
+		for _, v := range tt.values {
+			timer.Update(time.Duration(v))
+		}
+
+		snap := timer.Snapshot()
+
+		ps := snap.Percentiles([]float64{50, 95, 99})
+
+		val := snap.Values()
+
+		if len(val) > 0 {
+			if tt.wantMin != val[0] {
+				t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin)
+			}
+
+			if tt.wantMax != val[len(val)-1] {
+				t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax)
+			}
+		}
+
+		if tt.wantMean != snap.Mean() {
+			t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean)
+		}
+
+		if tt.wantP50 != ps[0] {
+			t.Fatalf("%d: p50: got %d, want %d", ind, ps[0], tt.wantP50)
+		}
+
+		if tt.wantP95 != ps[1] {
+			t.Fatalf("%d: p95: got %d, want %d", ind, ps[1], tt.wantP95)
+		}
+
+		if tt.wantP99 != ps[2] {
+			t.Fatalf("%d: p99: got %d, want %d", ind, ps[2], tt.wantP99)
+		}
+	}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/runtime.go b/metrics/runtime.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/runtime.go
rename to metrics/runtime.go
index 11c6b785a0..9450c479ba 100644
--- a/vendor/github.com/rcrowley/go-metrics/runtime.go
+++ b/metrics/runtime.go
@@ -55,7 +55,7 @@ var (
 // Capture new values for the Go runtime statistics exported in
 // runtime.MemStats.  This is designed to be called as a goroutine.
 func CaptureRuntimeMemStats(r Registry, d time.Duration) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		CaptureRuntimeMemStatsOnce(r)
 	}
 }
diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go b/metrics/runtime_cgo.go
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/runtime_cgo.go
rename to metrics/runtime_cgo.go
diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go b/metrics/runtime_gccpufraction.go
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go
rename to metrics/runtime_gccpufraction.go
diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go
rename to metrics/runtime_no_cgo.go
diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go b/metrics/runtime_no_gccpufraction.go
similarity index 100%
rename from vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go
rename to metrics/runtime_no_gccpufraction.go
diff --git a/metrics/runtime_test.go b/metrics/runtime_test.go
new file mode 100644
index 0000000000..ebbfd501a2
--- /dev/null
+++ b/metrics/runtime_test.go
@@ -0,0 +1,88 @@
+package metrics
+
+import (
+	"runtime"
+	"testing"
+	"time"
+)
+
+func BenchmarkRuntimeMemStats(b *testing.B) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		CaptureRuntimeMemStatsOnce(r)
+	}
+}
+
+func TestRuntimeMemStats(t *testing.T) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	CaptureRuntimeMemStatsOnce(r)
+	zero := runtimeMetrics.MemStats.PauseNs.Count() // Get a "zero" since GC may have run before these tests.
+	runtime.GC()
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 1 != count-zero {
+		t.Fatal(count - zero)
+	}
+	runtime.GC()
+	runtime.GC()
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 3 != count-zero {
+		t.Fatal(count - zero)
+	}
+	for i := 0; i < 256; i++ {
+		runtime.GC()
+	}
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 259 != count-zero {
+		t.Fatal(count - zero)
+	}
+	for i := 0; i < 257; i++ {
+		runtime.GC()
+	}
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 515 != count-zero { // We lost one because there were too many GCs between captures.
+		t.Fatal(count - zero)
+	}
+}
+
+func TestRuntimeMemStatsNumThread(t *testing.T) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	CaptureRuntimeMemStatsOnce(r)
+
+	if value := runtimeMetrics.NumThread.Value(); value < 1 {
+		t.Fatalf("got NumThread: %d, wanted at least 1", value)
+	}
+}
+
+func TestRuntimeMemStatsBlocking(t *testing.T) {
+	if g := runtime.GOMAXPROCS(0); g < 2 {
+		t.Skipf("skipping TestRuntimeMemStatsBlocking with GOMAXPROCS=%d\n", g)
+	}
+	ch := make(chan int)
+	go testRuntimeMemStatsBlocking(ch)
+	var memStats runtime.MemStats
+	t0 := time.Now()
+	runtime.ReadMemStats(&memStats)
+	t1 := time.Now()
+	t.Log("i++ during runtime.ReadMemStats:", <-ch)
+	go testRuntimeMemStatsBlocking(ch)
+	d := t1.Sub(t0)
+	t.Log(d)
+	time.Sleep(d)
+	t.Log("i++ during time.Sleep:", <-ch)
+}
+
+func testRuntimeMemStatsBlocking(ch chan int) {
+	i := 0
+	for {
+		select {
+		case ch <- i:
+			return
+		default:
+			i++
+		}
+	}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/sample.go b/metrics/sample.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/sample.go
rename to metrics/sample.go
index fecee5ef68..5c4845a4f8 100644
--- a/vendor/github.com/rcrowley/go-metrics/sample.go
+++ b/metrics/sample.go
@@ -46,7 +46,7 @@ type ExpDecaySample struct {
 // NewExpDecaySample constructs a new exponentially-decaying sample with the
 // given reservoir size and alpha.
 func NewExpDecaySample(reservoirSize int, alpha float64) Sample {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilSample{}
 	}
 	s := &ExpDecaySample{
@@ -407,7 +407,7 @@ type UniformSample struct {
 // NewUniformSample constructs a new uniform sample with the given reservoir
 // size.
 func NewUniformSample(reservoirSize int) Sample {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilSample{}
 	}
 	return &UniformSample{
diff --git a/metrics/sample_test.go b/metrics/sample_test.go
new file mode 100644
index 0000000000..d60e99c5bb
--- /dev/null
+++ b/metrics/sample_test.go
@@ -0,0 +1,363 @@
+package metrics
+
+import (
+	"math/rand"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively
+// expensive computations like Variance, the cost of copying the Sample, as
+// approximated by a make and copy, is much greater than the cost of the
+// computation for small samples and only slightly less for large samples.
+func BenchmarkCompute1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCompute1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCopy1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+func BenchmarkCopy1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+
+func BenchmarkExpDecaySample257(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(257, 0.015))
+}
+
+func BenchmarkExpDecaySample514(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(514, 0.015))
+}
+
+func BenchmarkExpDecaySample1028(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(1028, 0.015))
+}
+
+func BenchmarkUniformSample257(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(257))
+}
+
+func BenchmarkUniformSample514(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(514))
+}
+
+func BenchmarkUniformSample1028(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(1028))
+}
+
+func TestExpDecaySample10(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 10; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 10 != size {
+		t.Errorf("s.Count(): 10 != %v\n", size)
+	}
+	if size := s.Size(); 10 != size {
+		t.Errorf("s.Size(): 10 != %v\n", size)
+	}
+	if l := len(s.Values()); 10 != l {
+		t.Errorf("len(s.Values()): 10 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 10 || v < 0 {
+			t.Errorf("out of range [0, 10): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample100(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(1000, 0.01)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 100 != size {
+		t.Errorf("s.Count(): 100 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 100 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample1000(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 1000): %v\n", v)
+		}
+	}
+}
+
+// This test makes sure that the sample's priority is not amplified by using
+// nanosecond duration since start rather than second duration since start.
+// The priority becomes +Inf quickly after starting if this is done,
+// effectively freezing the set of samples until a rescale step happens.
+func TestExpDecaySampleNanosecondRegression(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 100; i++ {
+		s.Update(10)
+	}
+	time.Sleep(1 * time.Millisecond)
+	for i := 0; i < 100; i++ {
+		s.Update(20)
+	}
+	v := s.Values()
+	avg := float64(0)
+	for i := 0; i < len(v); i++ {
+		avg += float64(v[i])
+	}
+	avg /= float64(len(v))
+	if avg > 16 || avg < 14 {
+		t.Errorf("out of range [14, 16]: %v\n", avg)
+	}
+}
+
+func TestExpDecaySampleRescale(t *testing.T) {
+	s := NewExpDecaySample(2, 0.001).(*ExpDecaySample)
+	s.update(time.Now(), 1)
+	s.update(time.Now().Add(time.Hour+time.Microsecond), 1)
+	for _, v := range s.values.Values() {
+		if v.k == 0.0 {
+			t.Fatal("v.k == 0.0")
+		}
+	}
+}
+
+func TestExpDecaySampleSnapshot(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testExpDecaySampleStatistics(t, snapshot)
+}
+
+func TestExpDecaySampleStatistics(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	testExpDecaySampleStatistics(t, s)
+}
+
+func TestUniformSample(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestUniformSampleIncludesTail(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	max := 100
+	for i := 0; i < max; i++ {
+		s.Update(int64(i))
+	}
+	v := s.Values()
+	sum := 0
+	exp := (max - 1) * max / 2
+	for i := 0; i < len(v); i++ {
+		sum += int(v[i])
+	}
+	if exp != sum {
+		t.Errorf("sum: %v != %v\n", exp, sum)
+	}
+}
+
+func TestUniformSampleSnapshot(t *testing.T) {
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testUniformSampleStatistics(t, snapshot)
+}
+
+func TestUniformSampleStatistics(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	testUniformSampleStatistics(t, s)
+}
+
+func benchmarkSample(b *testing.B, s Sample) {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	pauseTotalNs := memStats.PauseTotalNs
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		s.Update(1)
+	}
+	b.StopTimer()
+	runtime.GC()
+	runtime.ReadMemStats(&memStats)
+	b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N)
+}
+
+func testExpDecaySampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 107 != min {
+		t.Errorf("s.Min(): 107 != %v\n", min)
+	}
+	if max := s.Max(); 10000 != max {
+		t.Errorf("s.Max(): 10000 != %v\n", max)
+	}
+	if mean := s.Mean(); 4965.98 != mean {
+		t.Errorf("s.Mean(): 4965.98 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2959.825156930727 != stdDev {
+		t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4615 != ps[0] {
+		t.Errorf("median: 4615 != %v\n", ps[0])
+	}
+	if 7672 != ps[1] {
+		t.Errorf("75th percentile: 7672 != %v\n", ps[1])
+	}
+	if 9998.99 != ps[2] {
+		t.Errorf("99th percentile: 9998.99 != %v\n", ps[2])
+	}
+}
+
+func testUniformSampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 37 != min {
+		t.Errorf("s.Min(): 37 != %v\n", min)
+	}
+	if max := s.Max(); 9989 != max {
+		t.Errorf("s.Max(): 9989 != %v\n", max)
+	}
+	if mean := s.Mean(); 4748.14 != mean {
+		t.Errorf("s.Mean(): 4748.14 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2826.684117548333 != stdDev {
+		t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4599 != ps[0] {
+		t.Errorf("median: 4599 != %v\n", ps[0])
+	}
+	if 7380.5 != ps[1] {
+		t.Errorf("75th percentile: 7380.5 != %v\n", ps[1])
+	}
+	if 9986.429999999998 != ps[2] {
+		t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2])
+	}
+}
+
+// TestUniformSampleConcurrentUpdateCount would expose data race problems with
+// concurrent Update and Count calls on Sample when test is called with -race
+// argument
+func TestUniformSampleConcurrentUpdateCount(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	s := NewUniformSample(100)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	quit := make(chan struct{})
+	go func() {
+		t := time.NewTicker(10 * time.Millisecond)
+		for {
+			select {
+			case <-t.C:
+				s.Update(rand.Int63())
+			case <-quit:
+				t.Stop()
+				return
+			}
+		}
+	}()
+	for i := 0; i < 1000; i++ {
+		s.Count()
+		time.Sleep(5 * time.Millisecond)
+	}
+	quit <- struct{}{}
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/syslog.go b/metrics/syslog.go
similarity index 98%
rename from vendor/github.com/rcrowley/go-metrics/syslog.go
rename to metrics/syslog.go
index 693f190855..a0ed4b1b23 100644
--- a/vendor/github.com/rcrowley/go-metrics/syslog.go
+++ b/metrics/syslog.go
@@ -11,7 +11,7 @@ import (
 // Output each metric in the given registry to syslog periodically using
 // the given syslogger.
 func Syslog(r Registry, d time.Duration, w *syslog.Writer) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		r.Each(func(name string, i interface{}) {
 			switch metric := i.(type) {
 			case Counter:
diff --git a/vendor/github.com/rcrowley/go-metrics/timer.go b/metrics/timer.go
similarity index 93%
rename from vendor/github.com/rcrowley/go-metrics/timer.go
rename to metrics/timer.go
index 17db8f8d20..89e22208fd 100644
--- a/vendor/github.com/rcrowley/go-metrics/timer.go
+++ b/metrics/timer.go
@@ -19,6 +19,7 @@ type Timer interface {
 	RateMean() float64
 	Snapshot() Timer
 	StdDev() float64
+	Stop()
 	Sum() int64
 	Time(func())
 	Update(time.Duration)
@@ -28,6 +29,8 @@ type Timer interface {
 
 // GetOrRegisterTimer returns an existing Timer or constructs and registers a
 // new StandardTimer.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func GetOrRegisterTimer(name string, r Registry) Timer {
 	if nil == r {
 		r = DefaultRegistry
@@ -36,8 +39,9 @@ func GetOrRegisterTimer(name string, r Registry) Timer {
 }
 
 // NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter.
+// Be sure to call Stop() once the timer is of no use to allow for garbage collection.
 func NewCustomTimer(h Histogram, m Meter) Timer {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilTimer{}
 	}
 	return &StandardTimer{
@@ -47,6 +51,8 @@ func NewCustomTimer(h Histogram, m Meter) Timer {
 }
 
 // NewRegisteredTimer constructs and registers a new StandardTimer.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func NewRegisteredTimer(name string, r Registry) Timer {
 	c := NewTimer()
 	if nil == r {
@@ -58,8 +64,9 @@ func NewRegisteredTimer(name string, r Registry) Timer {
 
 // NewTimer constructs a new StandardTimer using an exponentially-decaying
 // sample with the same reservoir size and alpha as UNIX load averages.
+// Be sure to call Stop() once the timer is of no use to allow for garbage collection.
 func NewTimer() Timer {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilTimer{}
 	}
 	return &StandardTimer{
@@ -112,6 +119,9 @@ func (NilTimer) Snapshot() Timer { return NilTimer{} }
 // StdDev is a no-op.
 func (NilTimer) StdDev() float64 { return 0.0 }
 
+// Stop is a no-op.
+func (NilTimer) Stop() {}
+
 // Sum is a no-op.
 func (NilTimer) Sum() int64 { return 0 }
 
@@ -201,6 +211,11 @@ func (t *StandardTimer) StdDev() float64 {
 	return t.histogram.StdDev()
 }
 
+// Stop stops the meter.
+func (t *StandardTimer) Stop() {
+	t.meter.Stop()
+}
+
 // Sum returns the sum in the sample.
 func (t *StandardTimer) Sum() int64 {
 	return t.histogram.Sum()
@@ -288,6 +303,9 @@ func (t *TimerSnapshot) Snapshot() Timer { return t }
 // was taken.
 func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() }
 
+// Stop is a no-op.
+func (t *TimerSnapshot) Stop() {}
+
 // Sum returns the sum at the time the snapshot was taken.
 func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() }
 
diff --git a/metrics/timer_test.go b/metrics/timer_test.go
new file mode 100644
index 0000000000..f85c9b803d
--- /dev/null
+++ b/metrics/timer_test.go
@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"fmt"
+	"math"
+	"testing"
+	"time"
+)
+
+func BenchmarkTimer(b *testing.B) {
+	tm := NewTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		tm.Update(1)
+	}
+}
+
+func TestGetOrRegisterTimer(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredTimer("foo", r).Update(47)
+	if tm := GetOrRegisterTimer("foo", r); 1 != tm.Count() {
+		t.Fatal(tm)
+	}
+}
+
+func TestTimerExtremes(t *testing.T) {
+	tm := NewTimer()
+	tm.Update(math.MaxInt64)
+	tm.Update(0)
+	if stdDev := tm.StdDev(); 4.611686018427388e+18 != stdDev {
+		t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev)
+	}
+}
+
+func TestTimerStop(t *testing.T) {
+	l := len(arbiter.meters)
+	tm := NewTimer()
+	if len(arbiter.meters) != l+1 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters))
+	}
+	tm.Stop()
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters))
+	}
+}
+
+func TestTimerFunc(t *testing.T) {
+	tm := NewTimer()
+	tm.Time(func() { time.Sleep(50e6) })
+	if max := tm.Max(); 45e6 > max || max > 55e6 {
+		t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max)
+	}
+}
+
+func TestTimerZero(t *testing.T) {
+	tm := NewTimer()
+	if count := tm.Count(); 0 != count {
+		t.Errorf("tm.Count(): 0 != %v\n", count)
+	}
+	if min := tm.Min(); 0 != min {
+		t.Errorf("tm.Min(): 0 != %v\n", min)
+	}
+	if max := tm.Max(); 0 != max {
+		t.Errorf("tm.Max(): 0 != %v\n", max)
+	}
+	if mean := tm.Mean(); 0.0 != mean {
+		t.Errorf("tm.Mean(): 0.0 != %v\n", mean)
+	}
+	if stdDev := tm.StdDev(); 0.0 != stdDev {
+		t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev)
+	}
+	ps := tm.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 0.0 != ps[0] {
+		t.Errorf("median: 0.0 != %v\n", ps[0])
+	}
+	if 0.0 != ps[1] {
+		t.Errorf("75th percentile: 0.0 != %v\n", ps[1])
+	}
+	if 0.0 != ps[2] {
+		t.Errorf("99th percentile: 0.0 != %v\n", ps[2])
+	}
+	if rate1 := tm.Rate1(); 0.0 != rate1 {
+		t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1)
+	}
+	if rate5 := tm.Rate5(); 0.0 != rate5 {
+		t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5)
+	}
+	if rate15 := tm.Rate15(); 0.0 != rate15 {
+		t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15)
+	}
+	if rateMean := tm.RateMean(); 0.0 != rateMean {
+		t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean)
+	}
+}
+
+func ExampleGetOrRegisterTimer() {
+	m := "account.create.latency"
+	t := GetOrRegisterTimer(m, nil)
+	t.Update(47)
+	fmt.Println(t.Max()) // Output: 47
+}
diff --git a/vendor/github.com/rcrowley/go-metrics/validate.sh b/metrics/validate.sh
similarity index 93%
rename from vendor/github.com/rcrowley/go-metrics/validate.sh
rename to metrics/validate.sh
index f6499982e5..c4ae91e642 100755
--- a/vendor/github.com/rcrowley/go-metrics/validate.sh
+++ b/metrics/validate.sh
@@ -7,4 +7,4 @@ GOFMT_LINES=`gofmt -l . | wc -l | xargs`
 test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues"
 
 # run the tests for the root package
-go test .
+go test -race .
diff --git a/vendor/github.com/rcrowley/go-metrics/writer.go b/metrics/writer.go
similarity index 99%
rename from vendor/github.com/rcrowley/go-metrics/writer.go
rename to metrics/writer.go
index 091e971d2e..88521a80d9 100644
--- a/vendor/github.com/rcrowley/go-metrics/writer.go
+++ b/metrics/writer.go
@@ -10,7 +10,7 @@ import (
 // Write sorts writes each metric in the given registry periodically to the
 // given io.Writer.
 func Write(r Registry, d time.Duration, w io.Writer) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		WriteOnce(r, w)
 	}
 }
diff --git a/metrics/writer_test.go b/metrics/writer_test.go
new file mode 100644
index 0000000000..1aacc28712
--- /dev/null
+++ b/metrics/writer_test.go
@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestMetricsSorting(t *testing.T) {
+	var namedMetrics = namedMetricSlice{
+		{name: "zzz"},
+		{name: "bbb"},
+		{name: "fff"},
+		{name: "ggg"},
+	}
+
+	sort.Sort(namedMetrics)
+	for i, name := range []string{"bbb", "fff", "ggg", "zzz"} {
+		if namedMetrics[i].name != name {
+			t.Fail()
+		}
+	}
+}
diff --git a/mobile/bind.go b/mobile/bind.go
index 7a1bf9e605..d6e621a258 100644
--- a/mobile/bind.go
+++ b/mobile/bind.go
@@ -154,12 +154,20 @@ func (c *BoundContract) GetDeployer() *Transaction {
 // Call invokes the (constant) contract method with params as input values and
 // sets the output to result.
 func (c *BoundContract) Call(opts *CallOpts, out *Interfaces, method string, args *Interfaces) error {
-	results := make([]interface{}, len(out.objects))
-	copy(results, out.objects)
-	if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil {
-		return err
+	if len(out.objects) == 1 {
+		result := out.objects[0]
+		if err := c.contract.Call(&opts.opts, result, method, args.objects...); err != nil {
+			return err
+		}
+		out.objects[0] = result
+	} else {
+		results := make([]interface{}, len(out.objects))
+		copy(results, out.objects)
+		if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil {
+			return err
+		}
+		copy(out.objects, results)
 	}
-	copy(out.objects, results)
 	return nil
 }
 
diff --git a/node/api.go b/node/api.go
index 1b04b70938..a3b8bc0bb5 100644
--- a/node/api.go
+++ b/node/api.go
@@ -24,10 +24,10 @@ import (
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/rcrowley/go-metrics"
 )
 
 // PrivateAdminAPI is the collection of administrative API methods exposed only
@@ -114,7 +114,7 @@ func (api *PrivateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription,
 }
 
 // StartRPC starts the HTTP RPC API server.
-func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string) (bool, error) {
+func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) {
 	api.node.lock.Lock()
 	defer api.node.lock.Unlock()
 
@@ -141,6 +141,14 @@ func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis
 		}
 	}
 
+	allowedVHosts := api.node.config.HTTPVirtualHosts
+	if vhosts != nil {
+		allowedVHosts = nil
+		for _, vhost := range strings.Split(*host, ",") {
+			allowedVHosts = append(allowedVHosts, strings.TrimSpace(vhost))
+		}
+	}
+
 	modules := api.node.httpWhitelist
 	if apis != nil {
 		modules = nil
@@ -149,7 +157,7 @@ func (api *PrivateAdminAPI) StartRPC(host *string, port *int, cors *string, apis
 		}
 	}
 
-	if err := api.node.startHTTP(fmt.Sprintf("%s:%d", *host, *port), api.node.rpcAPIs, modules, allowedOrigins); err != nil {
+	if err := api.node.startHTTP(fmt.Sprintf("%s:%d", *host, *port), api.node.rpcAPIs, modules, allowedOrigins, allowedVHosts); err != nil {
 		return false, err
 	}
 	return true, nil
@@ -300,6 +308,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 		// Fill the counter with the metric details, formatting if requested
 		if raw {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"AvgRate01Min": metric.Rate1(),
@@ -330,6 +343,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 			}
 		} else {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"Avg01Min": format(metric.Rate1()*60, metric.Rate1()),
diff --git a/node/config.go b/node/config.go
index 7a0c1688ec..dda24583ee 100644
--- a/node/config.go
+++ b/node/config.go
@@ -105,6 +105,15 @@ type Config struct {
 	// useless for custom HTTP clients.
 	HTTPCors []string `toml:",omitempty"`
 
+	// HTTPVirtualHosts is the list of virtual hostnames which are allowed on incoming requests.
+	// This is by default {'localhost'}. Using this prevents attacks like
+	// DNS rebinding, which bypasses SOP by simply masquerading as being within the same
+	// origin. These attacks do not utilize CORS, since they are not cross-domain.
+	// By explicitly checking the Host-header, the server will not allow requests
+	// made against the server with a malicious host domain.
+	// Requests using ip address directly are not affected
+	HTTPVirtualHosts []string `toml:",omitempty"`
+
 	// HTTPModules is a list of API modules to expose via the HTTP RPC interface.
 	// If the module list is empty, all RPC API endpoints designated public will be
 	// exposed.
@@ -137,7 +146,7 @@ type Config struct {
 	WSExposeAll bool `toml:",omitempty"`
 
 	// Logger is a custom logger to use with the p2p.Server.
-	Logger log.Logger
+	Logger log.Logger `toml:",omitempty"`
 }
 
 // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into
diff --git a/node/node.go b/node/node.go
index ff7258033e..b02aecfad1 100644
--- a/node/node.go
+++ b/node/node.go
@@ -263,7 +263,7 @@ func (n *Node) startRPC(services map[reflect.Type]Service) error {
 		n.stopInProc()
 		return err
 	}
-	if err := n.startHTTP(n.httpEndpoint, apis, n.config.HTTPModules, n.config.HTTPCors); err != nil {
+	if err := n.startHTTP(n.httpEndpoint, apis, n.config.HTTPModules, n.config.HTTPCors, n.config.HTTPVirtualHosts); err != nil {
 		n.stopIPC()
 		n.stopInProc()
 		return err
@@ -287,7 +287,7 @@ func (n *Node) startInProc(apis []rpc.API) error {
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return err
 		}
-		n.log.Debug(fmt.Sprintf("InProc registered %T under '%s'", api.Service, api.Namespace))
+		n.log.Debug("InProc registered", "service", api.Service, "namespace", api.Namespace)
 	}
 	n.inprocHandler = handler
 	return nil
@@ -313,7 +313,7 @@ func (n *Node) startIPC(apis []rpc.API) error {
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return err
 		}
-		n.log.Debug(fmt.Sprintf("IPC registered %T under '%s'", api.Service, api.Namespace))
+		n.log.Debug("IPC registered", "service", api.Service, "namespace", api.Namespace)
 	}
 	// All APIs registered, start the IPC listener
 	var (
@@ -324,7 +324,7 @@ func (n *Node) startIPC(apis []rpc.API) error {
 		return err
 	}
 	go func() {
-		n.log.Info(fmt.Sprintf("IPC endpoint opened: %s", n.ipcEndpoint))
+		n.log.Info("IPC endpoint opened", "url", n.ipcEndpoint)
 
 		for {
 			conn, err := listener.Accept()
@@ -337,7 +337,7 @@ func (n *Node) startIPC(apis []rpc.API) error {
 					return
 				}
 				// Not closed, just some error; report and continue
-				n.log.Error(fmt.Sprintf("IPC accept failed: %v", err))
+				n.log.Error("IPC accept failed", "err", err)
 				continue
 			}
 			go handler.ServeCodec(rpc.NewJSONCodec(conn), rpc.OptionMethodInvocation|rpc.OptionSubscriptions)
@@ -356,7 +356,7 @@ func (n *Node) stopIPC() {
 		n.ipcListener.Close()
 		n.ipcListener = nil
 
-		n.log.Info(fmt.Sprintf("IPC endpoint closed: %s", n.ipcEndpoint))
+		n.log.Info("IPC endpoint closed", "endpoint", n.ipcEndpoint)
 	}
 	if n.ipcHandler != nil {
 		n.ipcHandler.Stop()
@@ -365,7 +365,7 @@ func (n *Node) stopIPC() {
 }
 
 // startHTTP initializes and starts the HTTP RPC endpoint.
-func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors []string) error {
+func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string) error {
 	// Short circuit if the HTTP endpoint isn't being exposed
 	if endpoint == "" {
 		return nil
@@ -382,7 +382,7 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 				return err
 			}
-			n.log.Debug(fmt.Sprintf("HTTP registered %T under '%s'", api.Service, api.Namespace))
+			n.log.Debug("HTTP registered", "service", api.Service, "namespace", api.Namespace)
 		}
 	}
 	// All APIs registered, start the HTTP listener
@@ -393,9 +393,8 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 	if listener, err = net.Listen("tcp", endpoint); err != nil {
 		return err
 	}
-	go rpc.NewHTTPServer(cors, handler).Serve(listener)
-	n.log.Info(fmt.Sprintf("HTTP endpoint opened: http://%s", endpoint))
-
+	go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener)
+	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ","))
 	// All listeners booted successfully
 	n.httpEndpoint = endpoint
 	n.httpListener = listener
@@ -410,7 +409,7 @@ func (n *Node) stopHTTP() {
 		n.httpListener.Close()
 		n.httpListener = nil
 
-		n.log.Info(fmt.Sprintf("HTTP endpoint closed: http://%s", n.httpEndpoint))
+		n.log.Info("HTTP endpoint closed", "url", fmt.Sprintf("http://%s", n.httpEndpoint))
 	}
 	if n.httpHandler != nil {
 		n.httpHandler.Stop()
@@ -436,7 +435,7 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig
 			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 				return err
 			}
-			n.log.Debug(fmt.Sprintf("WebSocket registered %T under '%s'", api.Service, api.Namespace))
+			n.log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
 		}
 	}
 	// All APIs registered, start the HTTP listener
@@ -448,7 +447,7 @@ func (n *Node) startWS(endpoint string, apis []rpc.API, modules []string, wsOrig
 		return err
 	}
 	go rpc.NewWSServer(wsOrigins, handler).Serve(listener)
-	n.log.Info(fmt.Sprintf("WebSocket endpoint opened: ws://%s", listener.Addr()))
+	n.log.Info("WebSocket endpoint opened", "url", fmt.Sprintf("ws://%s", listener.Addr()))
 
 	// All listeners booted successfully
 	n.wsEndpoint = endpoint
@@ -464,7 +463,7 @@ func (n *Node) stopWS() {
 		n.wsListener.Close()
 		n.wsListener = nil
 
-		n.log.Info(fmt.Sprintf("WebSocket endpoint closed: ws://%s", n.wsEndpoint))
+		n.log.Info("WebSocket endpoint closed", "url", fmt.Sprintf("ws://%s", n.wsEndpoint))
 	}
 	if n.wsHandler != nil {
 		n.wsHandler.Stop()
diff --git a/p2p/dial.go b/p2p/dial.go
index f5ff2c2117..d8feceb9f3 100644
--- a/p2p/dial.go
+++ b/p2p/dial.go
@@ -154,6 +154,9 @@ func (s *dialstate) addStatic(n *discover.Node) {
 func (s *dialstate) removeStatic(n *discover.Node) {
 	// This removes a task so future attempts to connect will not be made.
 	delete(s.static, n.ID)
+	// This removes a previous dial timestamp so that application
+	// can force a server to reconnect with chosen peer immediately.
+	s.hist.remove(n.ID)
 }
 
 func (s *dialstate) newTasks(nRunning int, peers map[discover.NodeID]*Peer, now time.Time) []task {
@@ -390,6 +393,16 @@ func (h dialHistory) min() pastDial {
 }
 func (h *dialHistory) add(id discover.NodeID, exp time.Time) {
 	heap.Push(h, pastDial{id, exp})
+
+}
+func (h *dialHistory) remove(id discover.NodeID) bool {
+	for i, v := range *h {
+		if v.id == id {
+			heap.Remove(h, i)
+			return true
+		}
+	}
+	return false
 }
 func (h dialHistory) contains(id discover.NodeID) bool {
 	for _, v := range h {
diff --git a/p2p/dial_test.go b/p2p/dial_test.go
index ad18ef9abe..2a7941fc65 100644
--- a/p2p/dial_test.go
+++ b/p2p/dial_test.go
@@ -515,6 +515,50 @@ func TestDialStateStaticDial(t *testing.T) {
 	})
 }
 
+// This test checks that static peers will be redialed immediately if they were re-added to a static list.
+func TestDialStaticAfterReset(t *testing.T) {
+	wantStatic := []*discover.Node{
+		{ID: uintID(1)},
+		{ID: uintID(2)},
+	}
+
+	rounds := []round{
+		// Static dials are launched for the nodes that aren't yet connected.
+		{
+			peers: nil,
+			new: []task{
+				&dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(1)}},
+				&dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(2)}},
+			},
+		},
+		// No new dial tasks, all peers are connected.
+		{
+			peers: []*Peer{
+				{rw: &conn{flags: staticDialedConn, id: uintID(1)}},
+				{rw: &conn{flags: staticDialedConn, id: uintID(2)}},
+			},
+			done: []task{
+				&dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(1)}},
+				&dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(2)}},
+			},
+			new: []task{
+				&waitExpireTask{Duration: 30 * time.Second},
+			},
+		},
+	}
+	dTest := dialtest{
+		init:   newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		rounds: rounds,
+	}
+	runDialTest(t, dTest)
+	for _, n := range wantStatic {
+		dTest.init.removeStatic(n)
+		dTest.init.addStatic(n)
+	}
+	// without removing peers they will be considered recently dialed
+	runDialTest(t, dTest)
+}
+
 // This test checks that past dials are not retried for some time.
 func TestDialStateCache(t *testing.T) {
 	wantStatic := []*discover.Node{
diff --git a/p2p/discover/database.go b/p2p/discover/database.go
index b136609f24..6f98de9b42 100644
--- a/p2p/discover/database.go
+++ b/p2p/discover/database.go
@@ -257,7 +257,7 @@ func (db *nodeDB) expireNodes() error {
 		}
 		// Skip the node if not expired yet (and not self)
 		if !bytes.Equal(id[:], db.self[:]) {
-			if seen := db.lastPong(id); seen.After(threshold) {
+			if seen := db.bondTime(id); seen.After(threshold) {
 				continue
 			}
 		}
@@ -278,13 +278,18 @@ func (db *nodeDB) updateLastPing(id NodeID, instance time.Time) error {
 	return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix())
 }
 
-// lastPong retrieves the time of the last successful contact from remote node.
-func (db *nodeDB) lastPong(id NodeID) time.Time {
+// bondTime retrieves the time of the last successful pong from remote node.
+func (db *nodeDB) bondTime(id NodeID) time.Time {
 	return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0)
 }
 
-// updateLastPong updates the last time a remote node successfully contacted.
-func (db *nodeDB) updateLastPong(id NodeID, instance time.Time) error {
+// hasBond reports whether the given node is considered bonded.
+func (db *nodeDB) hasBond(id NodeID) bool {
+	return time.Since(db.bondTime(id)) < nodeDBNodeExpiration
+}
+
+// updateBondTime updates the last pong time of a node.
+func (db *nodeDB) updateBondTime(id NodeID, instance time.Time) error {
 	return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix())
 }
 
@@ -327,7 +332,7 @@ seek:
 		if n.ID == db.self {
 			continue seek
 		}
-		if now.Sub(db.lastPong(n.ID)) > maxAge {
+		if now.Sub(db.bondTime(n.ID)) > maxAge {
 			continue seek
 		}
 		for i := range nodes {
diff --git a/p2p/discover/database_test.go b/p2p/discover/database_test.go
index be972fd2c3..c4fa44d099 100644
--- a/p2p/discover/database_test.go
+++ b/p2p/discover/database_test.go
@@ -125,13 +125,13 @@ func TestNodeDBFetchStore(t *testing.T) {
 		t.Errorf("ping: value mismatch: have %v, want %v", stored, inst)
 	}
 	// Check fetch/store operations on a node pong object
-	if stored := db.lastPong(node.ID); stored.Unix() != 0 {
+	if stored := db.bondTime(node.ID); stored.Unix() != 0 {
 		t.Errorf("pong: non-existing object: %v", stored)
 	}
-	if err := db.updateLastPong(node.ID, inst); err != nil {
+	if err := db.updateBondTime(node.ID, inst); err != nil {
 		t.Errorf("pong: failed to update: %v", err)
 	}
-	if stored := db.lastPong(node.ID); stored.Unix() != inst.Unix() {
+	if stored := db.bondTime(node.ID); stored.Unix() != inst.Unix() {
 		t.Errorf("pong: value mismatch: have %v, want %v", stored, inst)
 	}
 	// Check fetch/store operations on a node findnode-failure object
@@ -224,8 +224,8 @@ func TestNodeDBSeedQuery(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to insert lastPong: %v", i, err)
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
+			t.Fatalf("node %d: failed to insert bondTime: %v", i, err)
 		}
 	}
 
@@ -332,8 +332,8 @@ func TestNodeDBExpiration(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to update pong: %v", i, err)
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
+			t.Fatalf("node %d: failed to update bondTime: %v", i, err)
 		}
 	}
 	// Expire some of them, and check the rest
@@ -365,8 +365,8 @@ func TestNodeDBSelfExpiration(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to update pong: %v", i, err)
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
+			t.Fatalf("node %d: failed to update bondTime: %v", i, err)
 		}
 	}
 	// Expire the nodes and make sure self has been evacuated too
diff --git a/p2p/discover/node.go b/p2p/discover/node.go
index fc928a91af..3b0c84115c 100644
--- a/p2p/discover/node.go
+++ b/p2p/discover/node.go
@@ -29,6 +29,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -51,9 +52,8 @@ type Node struct {
 	// with ID.
 	sha common.Hash
 
-	// whether this node is currently being pinged in order to replace
-	// it in a bucket
-	contested bool
+	// Time when the node was added to the table.
+	addedAt time.Time
 }
 
 // NewNode creates a new node. It is mostly meant to be used for
diff --git a/p2p/discover/table.go b/p2p/discover/table.go
index ec4eb94ad5..6509326e69 100644
--- a/p2p/discover/table.go
+++ b/p2p/discover/table.go
@@ -23,10 +23,11 @@
 package discover
 
 import (
-	"crypto/rand"
+	crand "crypto/rand"
 	"encoding/binary"
 	"errors"
 	"fmt"
+	mrand "math/rand"
 	"net"
 	"sort"
 	"sync"
@@ -35,29 +36,45 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
 
 const (
-	alpha      = 3  // Kademlia concurrency factor
-	bucketSize = 16 // Kademlia bucket size
-	hashBits   = len(common.Hash{}) * 8
-	nBuckets   = hashBits + 1 // Number of buckets
+	alpha           = 3  // Kademlia concurrency factor
+	bucketSize      = 16 // Kademlia bucket size
+	maxReplacements = 10 // Size of per-bucket replacement list
 
-	maxBondingPingPongs = 16
-	maxFindnodeFailures = 5
+	// We keep buckets for the upper 1/15 of distances because
+	// it's very unlikely we'll ever encounter a node that's closer.
+	hashBits          = len(common.Hash{}) * 8
+	nBuckets          = hashBits / 15       // Number of buckets
+	bucketMinDistance = hashBits - nBuckets // Log distance of closest bucket
 
-	autoRefreshInterval = 1 * time.Hour
-	seedCount           = 30
-	seedMaxAge          = 5 * 24 * time.Hour
+	// IP address limits.
+	bucketIPLimit, bucketSubnet = 2, 24 // at most 2 addresses from the same /24
+	tableIPLimit, tableSubnet   = 10, 24
+
+	maxBondingPingPongs = 16 // Limit on the number of concurrent ping/pong interactions
+	maxFindnodeFailures = 5  // Nodes exceeding this limit are dropped
+
+	refreshInterval    = 30 * time.Minute
+	revalidateInterval = 10 * time.Second
+	copyNodesInterval  = 30 * time.Second
+	seedMinTableTime   = 5 * time.Minute
+	seedCount          = 30
+	seedMaxAge         = 5 * 24 * time.Hour
 )
 
 type Table struct {
-	mutex   sync.Mutex        // protects buckets, their content, and nursery
+	mutex   sync.Mutex        // protects buckets, bucket content, nursery, rand
 	buckets [nBuckets]*bucket // index of known nodes by distance
 	nursery []*Node           // bootstrap nodes
-	db      *nodeDB           // database of known nodes
+	rand    *mrand.Rand       // source of randomness, periodically reseeded
+	ips     netutil.DistinctNetSet
 
+	db         *nodeDB // database of known nodes
 	refreshReq chan chan struct{}
+	initDone   chan struct{}
 	closeReq   chan struct{}
 	closed     chan struct{}
 
@@ -89,9 +106,13 @@ type transport interface {
 
 // bucket contains nodes, ordered by their last activity. the entry
 // that was most recently active is the first element in entries.
-type bucket struct{ entries []*Node }
+type bucket struct {
+	entries      []*Node // live entries, sorted by time of last contact
+	replacements []*Node // recently seen nodes to be used if revalidation fails
+	ips          netutil.DistinctNetSet
+}
 
-func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string) (*Table, error) {
+func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string, bootnodes []*Node) (*Table, error) {
 	// If no node database was given, use an in-memory one
 	db, err := newNodeDB(nodeDBPath, Version, ourID)
 	if err != nil {
@@ -104,19 +125,42 @@ func newTable(t transport, ourID NodeID, ourAddr *net.UDPAddr, nodeDBPath string
 		bonding:    make(map[NodeID]*bondproc),
 		bondslots:  make(chan struct{}, maxBondingPingPongs),
 		refreshReq: make(chan chan struct{}),
+		initDone:   make(chan struct{}),
 		closeReq:   make(chan struct{}),
 		closed:     make(chan struct{}),
+		rand:       mrand.New(mrand.NewSource(0)),
+		ips:        netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit},
+	}
+	if err := tab.setFallbackNodes(bootnodes); err != nil {
+		return nil, err
 	}
 	for i := 0; i < cap(tab.bondslots); i++ {
 		tab.bondslots <- struct{}{}
 	}
 	for i := range tab.buckets {
-		tab.buckets[i] = new(bucket)
+		tab.buckets[i] = &bucket{
+			ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit},
+		}
 	}
-	go tab.refreshLoop()
+	tab.seedRand()
+	tab.loadSeedNodes(false)
+	// Start the background expiration goroutine after loading seeds so that the search for
+	// seed nodes also considers older nodes that would otherwise be removed by the
+	// expiration.
+	tab.db.ensureExpirer()
+	go tab.loop()
 	return tab, nil
 }
 
+func (tab *Table) seedRand() {
+	var b [8]byte
+	crand.Read(b[:])
+
+	tab.mutex.Lock()
+	tab.rand.Seed(int64(binary.BigEndian.Uint64(b[:])))
+	tab.mutex.Unlock()
+}
+
 // Self returns the local node.
 // The returned node should not be modified by the caller.
 func (tab *Table) Self() *Node {
@@ -127,9 +171,12 @@ func (tab *Table) Self() *Node {
 // table. It will not write the same node more than once. The nodes in
 // the slice are copies and can be modified by the caller.
 func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
+	if !tab.isInitDone() {
+		return 0
+	}
 	tab.mutex.Lock()
 	defer tab.mutex.Unlock()
-	// TODO: tree-based buckets would help here
+
 	// Find all non-empty buckets and get a fresh slice of their entries.
 	var buckets [][]*Node
 	for _, b := range tab.buckets {
@@ -141,8 +188,8 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
 		return 0
 	}
 	// Shuffle the buckets.
-	for i := uint32(len(buckets)) - 1; i > 0; i-- {
-		j := randUint(i)
+	for i := len(buckets) - 1; i > 0; i-- {
+		j := tab.rand.Intn(len(buckets))
 		buckets[i], buckets[j] = buckets[j], buckets[i]
 	}
 	// Move head of each bucket into buf, removing buckets that become empty.
@@ -161,15 +208,6 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
 	return i + 1
 }
 
-func randUint(max uint32) uint32 {
-	if max == 0 {
-		return 0
-	}
-	var b [4]byte
-	rand.Read(b[:])
-	return binary.BigEndian.Uint32(b[:]) % max
-}
-
 // Close terminates the network listener and flushes the node database.
 func (tab *Table) Close() {
 	select {
@@ -180,16 +218,15 @@ func (tab *Table) Close() {
 	}
 }
 
-// SetFallbackNodes sets the initial points of contact. These nodes
+// setFallbackNodes sets the initial points of contact. These nodes
 // are used to connect to the network if the table is empty and there
 // are no known nodes in the database.
-func (tab *Table) SetFallbackNodes(nodes []*Node) error {
+func (tab *Table) setFallbackNodes(nodes []*Node) error {
 	for _, n := range nodes {
 		if err := n.validateComplete(); err != nil {
 			return fmt.Errorf("bad bootstrap/fallback node %q (%v)", n, err)
 		}
 	}
-	tab.mutex.Lock()
 	tab.nursery = make([]*Node, 0, len(nodes))
 	for _, n := range nodes {
 		cpy := *n
@@ -198,11 +235,19 @@ func (tab *Table) SetFallbackNodes(nodes []*Node) error {
 		cpy.sha = crypto.Keccak256Hash(n.ID[:])
 		tab.nursery = append(tab.nursery, &cpy)
 	}
-	tab.mutex.Unlock()
-	tab.refresh()
 	return nil
 }
 
+// isInitDone returns whether the table's initial seeding procedure has completed.
+func (tab *Table) isInitDone() bool {
+	select {
+	case <-tab.initDone:
+		return true
+	default:
+		return false
+	}
+}
+
 // Resolve searches for a specific node with the given ID.
 // It returns nil if the node could not be found.
 func (tab *Table) Resolve(targetID NodeID) *Node {
@@ -314,33 +359,49 @@ func (tab *Table) refresh() <-chan struct{} {
 	return done
 }
 
-// refreshLoop schedules doRefresh runs and coordinates shutdown.
-func (tab *Table) refreshLoop() {
+// loop schedules refresh, revalidate runs and coordinates shutdown.
+func (tab *Table) loop() {
 	var (
-		timer   = time.NewTicker(autoRefreshInterval)
-		waiting []chan struct{} // accumulates waiting callers while doRefresh runs
-		done    chan struct{}   // where doRefresh reports completion
+		revalidate     = time.NewTimer(tab.nextRevalidateTime())
+		refresh        = time.NewTicker(refreshInterval)
+		copyNodes      = time.NewTicker(copyNodesInterval)
+		revalidateDone = make(chan struct{})
+		refreshDone    = make(chan struct{})           // where doRefresh reports completion
+		waiting        = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs
 	)
+	defer refresh.Stop()
+	defer revalidate.Stop()
+	defer copyNodes.Stop()
+
+	// Start initial refresh.
+	go tab.doRefresh(refreshDone)
+
 loop:
 	for {
 		select {
-		case <-timer.C:
-			if done == nil {
-				done = make(chan struct{})
-				go tab.doRefresh(done)
+		case <-refresh.C:
+			tab.seedRand()
+			if refreshDone == nil {
+				refreshDone = make(chan struct{})
+				go tab.doRefresh(refreshDone)
 			}
 		case req := <-tab.refreshReq:
 			waiting = append(waiting, req)
-			if done == nil {
-				done = make(chan struct{})
-				go tab.doRefresh(done)
+			if refreshDone == nil {
+				refreshDone = make(chan struct{})
+				go tab.doRefresh(refreshDone)
 			}
-		case <-done:
+		case <-refreshDone:
 			for _, ch := range waiting {
 				close(ch)
 			}
-			waiting = nil
-			done = nil
+			waiting, refreshDone = nil, nil
+		case <-revalidate.C:
+			go tab.doRevalidate(revalidateDone)
+		case <-revalidateDone:
+			revalidate.Reset(tab.nextRevalidateTime())
+		case <-copyNodes.C:
+			go tab.copyBondedNodes()
 		case <-tab.closeReq:
 			break loop
 		}
@@ -349,8 +410,8 @@ loop:
 	if tab.net != nil {
 		tab.net.close()
 	}
-	if done != nil {
-		<-done
+	if refreshDone != nil {
+		<-refreshDone
 	}
 	for _, ch := range waiting {
 		close(ch)
@@ -365,38 +426,109 @@ loop:
 func (tab *Table) doRefresh(done chan struct{}) {
 	defer close(done)
 
+	// Load nodes from the database and insert
+	// them. This should yield a few previously seen nodes that are
+	// (hopefully) still alive.
+	tab.loadSeedNodes(true)
+
+	// Run self lookup to discover new neighbor nodes.
+	tab.lookup(tab.self.ID, false)
+
 	// The Kademlia paper specifies that the bucket refresh should
 	// perform a lookup in the least recently used bucket. We cannot
 	// adhere to this because the findnode target is a 512bit value
 	// (not hash-sized) and it is not easily possible to generate a
 	// sha3 preimage that falls into a chosen bucket.
-	// We perform a lookup with a random target instead.
-	var target NodeID
-	rand.Read(target[:])
-	result := tab.lookup(target, false)
-	if len(result) > 0 {
+	// We perform a few lookups with a random target instead.
+	for i := 0; i < 3; i++ {
+		var target NodeID
+		crand.Read(target[:])
+		tab.lookup(target, false)
+	}
+}
+
+func (tab *Table) loadSeedNodes(bond bool) {
+	seeds := tab.db.querySeeds(seedCount, seedMaxAge)
+	seeds = append(seeds, tab.nursery...)
+	if bond {
+		seeds = tab.bondall(seeds)
+	}
+	for i := range seeds {
+		seed := seeds[i]
+		age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.bondTime(seed.ID)) }}
+		log.Debug("Found seed node in database", "id", seed.ID, "addr", seed.addr(), "age", age)
+		tab.add(seed)
+	}
+}
+
+// doRevalidate checks that the last node in a random bucket is still live
+// and replaces or deletes the node if it isn't.
+func (tab *Table) doRevalidate(done chan<- struct{}) {
+	defer func() { done <- struct{}{} }()
+
+	last, bi := tab.nodeToRevalidate()
+	if last == nil {
+		// No non-empty bucket found.
 		return
 	}
 
-	// The table is empty. Load nodes from the database and insert
-	// them. This should yield a few previously seen nodes that are
-	// (hopefully) still alive.
-	seeds := tab.db.querySeeds(seedCount, seedMaxAge)
-	seeds = tab.bondall(append(seeds, tab.nursery...))
+	// Ping the selected node and wait for a pong.
+	err := tab.ping(last.ID, last.addr())
 
-	if len(seeds) == 0 {
-		log.Debug("No discv4 seed nodes found")
-	}
-	for _, n := range seeds {
-		age := log.Lazy{Fn: func() time.Duration { return time.Since(tab.db.lastPong(n.ID)) }}
-		log.Trace("Found seed node in database", "id", n.ID, "addr", n.addr(), "age", age)
-	}
 	tab.mutex.Lock()
-	tab.stuff(seeds)
-	tab.mutex.Unlock()
+	defer tab.mutex.Unlock()
+	b := tab.buckets[bi]
+	if err == nil {
+		// The node responded, move it to the front.
+		log.Debug("Revalidated node", "b", bi, "id", last.ID)
+		b.bump(last)
+		return
+	}
+	// No reply received, pick a replacement or delete the node if there aren't
+	// any replacements.
+	if r := tab.replace(b, last); r != nil {
+		log.Debug("Replaced dead node", "b", bi, "id", last.ID, "ip", last.IP, "r", r.ID, "rip", r.IP)
+	} else {
+		log.Debug("Removed dead node", "b", bi, "id", last.ID, "ip", last.IP)
+	}
+}
 
-	// Finally, do a self lookup to fill up the buckets.
-	tab.lookup(tab.self.ID, false)
+// nodeToRevalidate returns the last node in a random, non-empty bucket.
+func (tab *Table) nodeToRevalidate() (n *Node, bi int) {
+	tab.mutex.Lock()
+	defer tab.mutex.Unlock()
+
+	for _, bi = range tab.rand.Perm(len(tab.buckets)) {
+		b := tab.buckets[bi]
+		if len(b.entries) > 0 {
+			last := b.entries[len(b.entries)-1]
+			return last, bi
+		}
+	}
+	return nil, 0
+}
+
+func (tab *Table) nextRevalidateTime() time.Duration {
+	tab.mutex.Lock()
+	defer tab.mutex.Unlock()
+
+	return time.Duration(tab.rand.Int63n(int64(revalidateInterval)))
+}
+
+// copyBondedNodes adds nodes from the table to the database if they have been in the table
+// longer then minTableTime.
+func (tab *Table) copyBondedNodes() {
+	tab.mutex.Lock()
+	defer tab.mutex.Unlock()
+
+	now := time.Now()
+	for _, b := range tab.buckets {
+		for _, n := range b.entries {
+			if now.Sub(n.addedAt) >= seedMinTableTime {
+				tab.db.updateNode(n)
+			}
+		}
+	}
 }
 
 // closest returns the n nodes in the table that are closest to the
@@ -459,15 +591,14 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16
 	if id == tab.self.ID {
 		return nil, errors.New("is self")
 	}
-	// Retrieve a previously known node and any recent findnode failures
-	node, fails := tab.db.node(id), 0
-	if node != nil {
-		fails = tab.db.findFails(id)
+	if pinged && !tab.isInitDone() {
+		return nil, errors.New("still initializing")
 	}
-	// If the node is unknown (non-bonded) or failed (remotely unknown), bond from scratch
+	// Start bonding if we haven't seen this node for a while or if it failed findnode too often.
+	node, fails := tab.db.node(id), tab.db.findFails(id)
+	age := time.Since(tab.db.bondTime(id))
 	var result error
-	age := time.Since(tab.db.lastPong(id))
-	if node == nil || fails > 0 || age > nodeDBNodeExpiration {
+	if fails > 0 || age > nodeDBNodeExpiration {
 		log.Trace("Starting bonding ping/pong", "id", id, "known", node != nil, "failcount", fails, "age", age)
 
 		tab.bondmu.Lock()
@@ -494,10 +625,10 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16
 			node = w.n
 		}
 	}
+	// Add the node to the table even if the bonding ping/pong
+	// fails. It will be relaced quickly if it continues to be
+	// unresponsive.
 	if node != nil {
-		// Add the node to the table even if the bonding ping/pong
-		// fails. It will be relaced quickly if it continues to be
-		// unresponsive.
 		tab.add(node)
 		tab.db.updateFindFails(id, 0)
 	}
@@ -522,7 +653,6 @@ func (tab *Table) pingpong(w *bondproc, pinged bool, id NodeID, addr *net.UDPAdd
 	}
 	// Bonding succeeded, update the node database.
 	w.n = NewNode(id, addr.IP, uint16(addr.Port), tcpPort)
-	tab.db.updateNode(w.n)
 	close(w.done)
 }
 
@@ -533,17 +663,19 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error {
 	if err := tab.net.ping(id, addr); err != nil {
 		return err
 	}
-	tab.db.updateLastPong(id, time.Now())
-
-	// Start the background expiration goroutine after the first
-	// successful communication. Subsequent calls have no effect if it
-	// is already running. We do this here instead of somewhere else
-	// so that the search for seed nodes also considers older nodes
-	// that would otherwise be removed by the expiration.
-	tab.db.ensureExpirer()
+	tab.db.updateBondTime(id, time.Now())
 	return nil
 }
 
+// bucket returns the bucket for the given node ID hash.
+func (tab *Table) bucket(sha common.Hash) *bucket {
+	d := logdist(tab.self.sha, sha)
+	if d <= bucketMinDistance {
+		return tab.buckets[0]
+	}
+	return tab.buckets[d-bucketMinDistance-1]
+}
+
 // add attempts to add the given node its corresponding bucket. If the
 // bucket has space available, adding the node succeeds immediately.
 // Otherwise, the node is added if the least recently active node in
@@ -551,57 +683,29 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error {
 //
 // The caller must not hold tab.mutex.
 func (tab *Table) add(new *Node) {
-	b := tab.buckets[logdist(tab.self.sha, new.sha)]
 	tab.mutex.Lock()
 	defer tab.mutex.Unlock()
-	if b.bump(new) {
-		return
-	}
-	var oldest *Node
-	if len(b.entries) == bucketSize {
-		oldest = b.entries[bucketSize-1]
-		if oldest.contested {
-			// The node is already being replaced, don't attempt
-			// to replace it.
-			return
-		}
-		oldest.contested = true
-		// Let go of the mutex so other goroutines can access
-		// the table while we ping the least recently active node.
-		tab.mutex.Unlock()
-		err := tab.ping(oldest.ID, oldest.addr())
-		tab.mutex.Lock()
-		oldest.contested = false
-		if err == nil {
-			// The node responded, don't replace it.
-			return
-		}
-	}
-	added := b.replace(new, oldest)
-	if added && tab.nodeAddedHook != nil {
-		tab.nodeAddedHook(new)
+
+	b := tab.bucket(new.sha)
+	if !tab.bumpOrAdd(b, new) {
+		// Node is not in table. Add it to the replacement list.
+		tab.addReplacement(b, new)
 	}
 }
 
 // stuff adds nodes the table to the end of their corresponding bucket
-// if the bucket is not full. The caller must hold tab.mutex.
+// if the bucket is not full. The caller must not hold tab.mutex.
 func (tab *Table) stuff(nodes []*Node) {
-outer:
+	tab.mutex.Lock()
+	defer tab.mutex.Unlock()
+
 	for _, n := range nodes {
 		if n.ID == tab.self.ID {
 			continue // don't add self
 		}
-		bucket := tab.buckets[logdist(tab.self.sha, n.sha)]
-		for i := range bucket.entries {
-			if bucket.entries[i].ID == n.ID {
-				continue outer // already in bucket
-			}
-		}
-		if len(bucket.entries) < bucketSize {
-			bucket.entries = append(bucket.entries, n)
-			if tab.nodeAddedHook != nil {
-				tab.nodeAddedHook(n)
-			}
+		b := tab.bucket(n.sha)
+		if len(b.entries) < bucketSize {
+			tab.bumpOrAdd(b, n)
 		}
 	}
 }
@@ -611,36 +715,72 @@ outer:
 func (tab *Table) delete(node *Node) {
 	tab.mutex.Lock()
 	defer tab.mutex.Unlock()
-	bucket := tab.buckets[logdist(tab.self.sha, node.sha)]
-	for i := range bucket.entries {
-		if bucket.entries[i].ID == node.ID {
-			bucket.entries = append(bucket.entries[:i], bucket.entries[i+1:]...)
-			return
-		}
-	}
+
+	tab.deleteInBucket(tab.bucket(node.sha), node)
 }
 
-func (b *bucket) replace(n *Node, last *Node) bool {
-	// Don't add if b already contains n.
-	for i := range b.entries {
-		if b.entries[i].ID == n.ID {
-			return false
-		}
+func (tab *Table) addIP(b *bucket, ip net.IP) bool {
+	if netutil.IsLAN(ip) {
+		return true
 	}
-	// Replace last if it is still the last entry or just add n if b
-	// isn't full. If is no longer the last entry, it has either been
-	// replaced with someone else or became active.
-	if len(b.entries) == bucketSize && (last == nil || b.entries[bucketSize-1].ID != last.ID) {
+	if !tab.ips.Add(ip) {
+		log.Debug("IP exceeds table limit", "ip", ip)
 		return false
 	}
-	if len(b.entries) < bucketSize {
-		b.entries = append(b.entries, nil)
+	if !b.ips.Add(ip) {
+		log.Debug("IP exceeds bucket limit", "ip", ip)
+		tab.ips.Remove(ip)
+		return false
 	}
-	copy(b.entries[1:], b.entries)
-	b.entries[0] = n
 	return true
 }
 
+func (tab *Table) removeIP(b *bucket, ip net.IP) {
+	if netutil.IsLAN(ip) {
+		return
+	}
+	tab.ips.Remove(ip)
+	b.ips.Remove(ip)
+}
+
+func (tab *Table) addReplacement(b *bucket, n *Node) {
+	for _, e := range b.replacements {
+		if e.ID == n.ID {
+			return // already in list
+		}
+	}
+	if !tab.addIP(b, n.IP) {
+		return
+	}
+	var removed *Node
+	b.replacements, removed = pushNode(b.replacements, n, maxReplacements)
+	if removed != nil {
+		tab.removeIP(b, removed.IP)
+	}
+}
+
+// replace removes n from the replacement list and replaces 'last' with it if it is the
+// last entry in the bucket. If 'last' isn't the last entry, it has either been replaced
+// with someone else or became active.
+func (tab *Table) replace(b *bucket, last *Node) *Node {
+	if len(b.entries) == 0 || b.entries[len(b.entries)-1].ID != last.ID {
+		// Entry has moved, don't replace it.
+		return nil
+	}
+	// Still the last entry.
+	if len(b.replacements) == 0 {
+		tab.deleteInBucket(b, last)
+		return nil
+	}
+	r := b.replacements[tab.rand.Intn(len(b.replacements))]
+	b.replacements = deleteNode(b.replacements, r)
+	b.entries[len(b.entries)-1] = r
+	tab.removeIP(b, last.IP)
+	return r
+}
+
+// bump moves the given node to the front of the bucket entry list
+// if it is contained in that list.
 func (b *bucket) bump(n *Node) bool {
 	for i := range b.entries {
 		if b.entries[i].ID == n.ID {
@@ -653,6 +793,50 @@ func (b *bucket) bump(n *Node) bool {
 	return false
 }
 
+// bumpOrAdd moves n to the front of the bucket entry list or adds it if the list isn't
+// full. The return value is true if n is in the bucket.
+func (tab *Table) bumpOrAdd(b *bucket, n *Node) bool {
+	if b.bump(n) {
+		return true
+	}
+	if len(b.entries) >= bucketSize || !tab.addIP(b, n.IP) {
+		return false
+	}
+	b.entries, _ = pushNode(b.entries, n, bucketSize)
+	b.replacements = deleteNode(b.replacements, n)
+	n.addedAt = time.Now()
+	if tab.nodeAddedHook != nil {
+		tab.nodeAddedHook(n)
+	}
+	return true
+}
+
+func (tab *Table) deleteInBucket(b *bucket, n *Node) {
+	b.entries = deleteNode(b.entries, n)
+	tab.removeIP(b, n.IP)
+}
+
+// pushNode adds n to the front of list, keeping at most max items.
+func pushNode(list []*Node, n *Node, max int) ([]*Node, *Node) {
+	if len(list) < max {
+		list = append(list, nil)
+	}
+	removed := list[len(list)-1]
+	copy(list[1:], list)
+	list[0] = n
+	return list, removed
+}
+
+// deleteNode removes n from list.
+func deleteNode(list []*Node, n *Node) []*Node {
+	for i := range list {
+		if list[i].ID == n.ID {
+			return append(list[:i], list[i+1:]...)
+		}
+	}
+	return list
+}
+
 // nodesByDistance is a list of nodes, ordered by
 // distance to target.
 type nodesByDistance struct {
diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go
index 1037cc6099..3ce48d2995 100644
--- a/p2p/discover/table_test.go
+++ b/p2p/discover/table_test.go
@@ -20,6 +20,7 @@ import (
 	"crypto/ecdsa"
 	"fmt"
 	"math/rand"
+	"sync"
 
 	"net"
 	"reflect"
@@ -32,60 +33,65 @@ import (
 )
 
 func TestTable_pingReplace(t *testing.T) {
-	doit := func(newNodeIsResponding, lastInBucketIsResponding bool) {
-		transport := newPingRecorder()
-		tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "")
-		defer tab.Close()
-		pingSender := NewNode(MustHexID("a502af0f59b2aab7746995408c79e9ca312d2793cc997e44fc55eda62f0150bbb8c59a6f9269ba3a081518b62699ee807c7c19c20125ddfccca872608af9e370"), net.IP{}, 99, 99)
-
-		// fill up the sender's bucket.
-		last := fillBucket(tab, 253)
-
-		// this call to bond should replace the last node
-		// in its bucket if the node is not responding.
-		transport.responding[last.ID] = lastInBucketIsResponding
-		transport.responding[pingSender.ID] = newNodeIsResponding
-		tab.bond(true, pingSender.ID, &net.UDPAddr{}, 0)
-
-		// first ping goes to sender (bonding pingback)
-		if !transport.pinged[pingSender.ID] {
-			t.Error("table did not ping back sender")
-		}
-		if newNodeIsResponding {
-			// second ping goes to oldest node in bucket
-			// to see whether it is still alive.
-			if !transport.pinged[last.ID] {
-				t.Error("table did not ping last node in bucket")
-			}
-		}
-
-		tab.mutex.Lock()
-		defer tab.mutex.Unlock()
-		if l := len(tab.buckets[253].entries); l != bucketSize {
-			t.Errorf("wrong bucket size after bond: got %d, want %d", l, bucketSize)
-		}
-
-		if lastInBucketIsResponding || !newNodeIsResponding {
-			if !contains(tab.buckets[253].entries, last.ID) {
-				t.Error("last entry was removed")
-			}
-			if contains(tab.buckets[253].entries, pingSender.ID) {
-				t.Error("new entry was added")
-			}
-		} else {
-			if contains(tab.buckets[253].entries, last.ID) {
-				t.Error("last entry was not removed")
-			}
-			if !contains(tab.buckets[253].entries, pingSender.ID) {
-				t.Error("new entry was not added")
-			}
-		}
+	run := func(newNodeResponding, lastInBucketResponding bool) {
+		name := fmt.Sprintf("newNodeResponding=%t/lastInBucketResponding=%t", newNodeResponding, lastInBucketResponding)
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			testPingReplace(t, newNodeResponding, lastInBucketResponding)
+		})
 	}
 
-	doit(true, true)
-	doit(false, true)
-	doit(true, false)
-	doit(false, false)
+	run(true, true)
+	run(false, true)
+	run(true, false)
+	run(false, false)
+}
+
+func testPingReplace(t *testing.T, newNodeIsResponding, lastInBucketIsResponding bool) {
+	transport := newPingRecorder()
+	tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil)
+	defer tab.Close()
+
+	// Wait for init so bond is accepted.
+	<-tab.initDone
+
+	// fill up the sender's bucket.
+	pingSender := NewNode(MustHexID("a502af0f59b2aab7746995408c79e9ca312d2793cc997e44fc55eda62f0150bbb8c59a6f9269ba3a081518b62699ee807c7c19c20125ddfccca872608af9e370"), net.IP{}, 99, 99)
+	last := fillBucket(tab, pingSender)
+
+	// this call to bond should replace the last node
+	// in its bucket if the node is not responding.
+	transport.dead[last.ID] = !lastInBucketIsResponding
+	transport.dead[pingSender.ID] = !newNodeIsResponding
+	tab.bond(true, pingSender.ID, &net.UDPAddr{}, 0)
+	tab.doRevalidate(make(chan struct{}, 1))
+
+	// first ping goes to sender (bonding pingback)
+	if !transport.pinged[pingSender.ID] {
+		t.Error("table did not ping back sender")
+	}
+	if !transport.pinged[last.ID] {
+		// second ping goes to oldest node in bucket
+		// to see whether it is still alive.
+		t.Error("table did not ping last node in bucket")
+	}
+
+	tab.mutex.Lock()
+	defer tab.mutex.Unlock()
+	wantSize := bucketSize
+	if !lastInBucketIsResponding && !newNodeIsResponding {
+		wantSize--
+	}
+	if l := len(tab.bucket(pingSender.sha).entries); l != wantSize {
+		t.Errorf("wrong bucket size after bond: got %d, want %d", l, wantSize)
+	}
+	if found := contains(tab.bucket(pingSender.sha).entries, last.ID); found != lastInBucketIsResponding {
+		t.Errorf("last entry found: %t, want: %t", found, lastInBucketIsResponding)
+	}
+	wantNewEntry := newNodeIsResponding && !lastInBucketIsResponding
+	if found := contains(tab.bucket(pingSender.sha).entries, pingSender.ID); found != wantNewEntry {
+		t.Errorf("new entry found: %t, want: %t", found, wantNewEntry)
+	}
 }
 
 func TestBucket_bumpNoDuplicates(t *testing.T) {
@@ -130,11 +136,45 @@ func TestBucket_bumpNoDuplicates(t *testing.T) {
 	}
 }
 
+// This checks that the table-wide IP limit is applied correctly.
+func TestTable_IPLimit(t *testing.T) {
+	transport := newPingRecorder()
+	tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil)
+	defer tab.Close()
+
+	for i := 0; i < tableIPLimit+1; i++ {
+		n := nodeAtDistance(tab.self.sha, i)
+		n.IP = net.IP{172, 0, 1, byte(i)}
+		tab.add(n)
+	}
+	if tab.len() > tableIPLimit {
+		t.Errorf("too many nodes in table")
+	}
+}
+
+// This checks that the table-wide IP limit is applied correctly.
+func TestTable_BucketIPLimit(t *testing.T) {
+	transport := newPingRecorder()
+	tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil)
+	defer tab.Close()
+
+	d := 3
+	for i := 0; i < bucketIPLimit+1; i++ {
+		n := nodeAtDistance(tab.self.sha, d)
+		n.IP = net.IP{172, 0, 1, byte(i)}
+		tab.add(n)
+	}
+	if tab.len() > bucketIPLimit {
+		t.Errorf("too many nodes in table")
+	}
+}
+
 // fillBucket inserts nodes into the given bucket until
 // it is full. The node's IDs dont correspond to their
 // hashes.
-func fillBucket(tab *Table, ld int) (last *Node) {
-	b := tab.buckets[ld]
+func fillBucket(tab *Table, n *Node) (last *Node) {
+	ld := logdist(tab.self.sha, n.sha)
+	b := tab.bucket(n.sha)
 	for len(b.entries) < bucketSize {
 		b.entries = append(b.entries, nodeAtDistance(tab.self.sha, ld))
 	}
@@ -146,30 +186,39 @@ func fillBucket(tab *Table, ld int) (last *Node) {
 func nodeAtDistance(base common.Hash, ld int) (n *Node) {
 	n = new(Node)
 	n.sha = hashAtDistance(base, ld)
-	n.IP = net.IP{10, 0, 2, byte(ld)}
+	n.IP = net.IP{byte(ld), 0, 2, byte(ld)}
 	copy(n.ID[:], n.sha[:]) // ensure the node still has a unique ID
 	return n
 }
 
-type pingRecorder struct{ responding, pinged map[NodeID]bool }
+type pingRecorder struct {
+	mu           sync.Mutex
+	dead, pinged map[NodeID]bool
+}
 
 func newPingRecorder() *pingRecorder {
-	return &pingRecorder{make(map[NodeID]bool), make(map[NodeID]bool)}
+	return &pingRecorder{
+		dead:   make(map[NodeID]bool),
+		pinged: make(map[NodeID]bool),
+	}
 }
 
 func (t *pingRecorder) findnode(toid NodeID, toaddr *net.UDPAddr, target NodeID) ([]*Node, error) {
-	panic("findnode called on pingRecorder")
+	return nil, nil
 }
 func (t *pingRecorder) close() {}
 func (t *pingRecorder) waitping(from NodeID) error {
 	return nil // remote always pings
 }
 func (t *pingRecorder) ping(toid NodeID, toaddr *net.UDPAddr) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	t.pinged[toid] = true
-	if t.responding[toid] {
-		return nil
-	} else {
+	if t.dead[toid] {
 		return errTimeout
+	} else {
+		return nil
 	}
 }
 
@@ -178,7 +227,8 @@ func TestTable_closest(t *testing.T) {
 
 	test := func(test *closeTest) bool {
 		// for any node table, Target and N
-		tab, _ := newTable(nil, test.Self, &net.UDPAddr{}, "")
+		transport := newPingRecorder()
+		tab, _ := newTable(transport, test.Self, &net.UDPAddr{}, "", nil)
 		defer tab.Close()
 		tab.stuff(test.All)
 
@@ -237,8 +287,11 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) {
 		},
 	}
 	test := func(buf []*Node) bool {
-		tab, _ := newTable(nil, NodeID{}, &net.UDPAddr{}, "")
+		transport := newPingRecorder()
+		tab, _ := newTable(transport, NodeID{}, &net.UDPAddr{}, "", nil)
 		defer tab.Close()
+		<-tab.initDone
+
 		for i := 0; i < len(buf); i++ {
 			ld := cfg.Rand.Intn(len(tab.buckets))
 			tab.stuff([]*Node{nodeAtDistance(tab.self.sha, ld)})
@@ -280,7 +333,7 @@ func (*closeTest) Generate(rand *rand.Rand, size int) reflect.Value {
 
 func TestTable_Lookup(t *testing.T) {
 	self := nodeAtDistance(common.Hash{}, 0)
-	tab, _ := newTable(lookupTestnet, self.ID, &net.UDPAddr{}, "")
+	tab, _ := newTable(lookupTestnet, self.ID, &net.UDPAddr{}, "", nil)
 	defer tab.Close()
 
 	// lookup on empty table returns no nodes
diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go
index 60436952d8..524c6e4988 100644
--- a/p2p/discover/udp.go
+++ b/p2p/discover/udp.go
@@ -216,9 +216,22 @@ type ReadPacket struct {
 	Addr *net.UDPAddr
 }
 
+// Config holds Table-related settings.
+type Config struct {
+	// These settings are required and configure the UDP listener:
+	PrivateKey *ecdsa.PrivateKey
+
+	// These settings are optional:
+	AnnounceAddr *net.UDPAddr      // local address announced in the DHT
+	NodeDBPath   string            // if set, the node database is stored at this filesystem location
+	NetRestrict  *netutil.Netlist  // network whitelist
+	Bootnodes    []*Node           // list of bootstrap nodes
+	Unhandled    chan<- ReadPacket // unhandled packets are sent on this channel
+}
+
 // ListenUDP returns a new table that listens for UDP packets on laddr.
-func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) {
-	tab, _, err := newUDP(priv, conn, realaddr, unhandled, nodeDBPath, netrestrict)
+func ListenUDP(c conn, cfg Config) (*Table, error) {
+	tab, _, err := newUDP(c, cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -226,25 +239,29 @@ func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandl
 	return tab, nil
 }
 
-func newUDP(priv *ecdsa.PrivateKey, c conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) {
+func newUDP(c conn, cfg Config) (*Table, *udp, error) {
 	udp := &udp{
 		conn:        c,
-		priv:        priv,
-		netrestrict: netrestrict,
+		priv:        cfg.PrivateKey,
+		netrestrict: cfg.NetRestrict,
 		closing:     make(chan struct{}),
 		gotreply:    make(chan reply),
 		addpending:  make(chan *pending),
 	}
+	realaddr := c.LocalAddr().(*net.UDPAddr)
+	if cfg.AnnounceAddr != nil {
+		realaddr = cfg.AnnounceAddr
+	}
 	// TODO: separate TCP port
 	udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port))
-	tab, err := newTable(udp, PubkeyID(&priv.PublicKey), realaddr, nodeDBPath)
+	tab, err := newTable(udp, PubkeyID(&cfg.PrivateKey.PublicKey), realaddr, cfg.NodeDBPath, cfg.Bootnodes)
 	if err != nil {
 		return nil, nil, err
 	}
 	udp.Table = tab
 
 	go udp.loop()
-	go udp.readLoop(unhandled)
+	go udp.readLoop(cfg.Unhandled)
 	return udp.Table, udp, nil
 }
 
@@ -256,14 +273,20 @@ func (t *udp) close() {
 
 // ping sends a ping message to the given node and waits for a reply.
 func (t *udp) ping(toid NodeID, toaddr *net.UDPAddr) error {
-	// TODO: maybe check for ReplyTo field in callback to measure RTT
-	errc := t.pending(toid, pongPacket, func(interface{}) bool { return true })
-	t.send(toaddr, pingPacket, &ping{
+	req := &ping{
 		Version:    Version,
 		From:       t.ourEndpoint,
 		To:         makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB
 		Expiration: uint64(time.Now().Add(expiration).Unix()),
+	}
+	packet, hash, err := encodePacket(t.priv, pingPacket, req)
+	if err != nil {
+		return err
+	}
+	errc := t.pending(toid, pongPacket, func(p interface{}) bool {
+		return bytes.Equal(p.(*pong).ReplyTok, hash)
 	})
+	t.write(toaddr, req.name(), packet)
 	return <-errc
 }
 
@@ -447,40 +470,45 @@ func init() {
 	}
 }
 
-func (t *udp) send(toaddr *net.UDPAddr, ptype byte, req packet) error {
-	packet, err := encodePacket(t.priv, ptype, req)
+func (t *udp) send(toaddr *net.UDPAddr, ptype byte, req packet) ([]byte, error) {
+	packet, hash, err := encodePacket(t.priv, ptype, req)
 	if err != nil {
-		return err
+		return hash, err
 	}
-	_, err = t.conn.WriteToUDP(packet, toaddr)
-	log.Trace(">> "+req.name(), "addr", toaddr, "err", err)
+	return hash, t.write(toaddr, req.name(), packet)
+}
+
+func (t *udp) write(toaddr *net.UDPAddr, what string, packet []byte) error {
+	_, err := t.conn.WriteToUDP(packet, toaddr)
+	log.Trace(">> "+what, "addr", toaddr, "err", err)
 	return err
 }
 
-func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) ([]byte, error) {
+func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet, hash []byte, err error) {
 	b := new(bytes.Buffer)
 	b.Write(headSpace)
 	b.WriteByte(ptype)
 	if err := rlp.Encode(b, req); err != nil {
 		log.Error("Can't encode discv4 packet", "err", err)
-		return nil, err
+		return nil, nil, err
 	}
-	packet := b.Bytes()
+	packet = b.Bytes()
 	sig, err := crypto.Sign(crypto.Keccak256(packet[headSize:]), priv)
 	if err != nil {
 		log.Error("Can't sign discv4 packet", "err", err)
-		return nil, err
+		return nil, nil, err
 	}
 	copy(packet[macSize:], sig)
 	// add the hash to the front. Note: this doesn't protect the
 	// packet in any way. Our public key will be part of this hash in
 	// The future.
-	copy(packet, crypto.Keccak256(packet[macSize:]))
-	return packet, nil
+	hash = crypto.Keccak256(packet[macSize:])
+	copy(packet, hash)
+	return packet, hash, nil
 }
 
 // readLoop runs in its own goroutine. it handles incoming UDP packets.
-func (t *udp) readLoop(unhandled chan ReadPacket) {
+func (t *udp) readLoop(unhandled chan<- ReadPacket) {
 	defer t.conn.Close()
 	if unhandled != nil {
 		defer close(unhandled)
@@ -585,7 +613,7 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte
 	if expired(req.Expiration) {
 		return errExpired
 	}
-	if t.db.node(fromID) == nil {
+	if !t.db.hasBond(fromID) {
 		// No bond exists, we don't process the packet. This prevents
 		// an attack vector where the discovery protocol could be used
 		// to amplify traffic in a DDOS attack. A malicious actor
@@ -601,18 +629,22 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte
 	t.mutex.Unlock()
 
 	p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())}
+	var sent bool
 	// Send neighbors in chunks with at most maxNeighbors per packet
 	// to stay below the 1280 byte limit.
-	for i, n := range closest {
-		if netutil.CheckRelayIP(from.IP, n.IP) != nil {
-			continue
+	for _, n := range closest {
+		if netutil.CheckRelayIP(from.IP, n.IP) == nil {
+			p.Nodes = append(p.Nodes, nodeToRPC(n))
 		}
-		p.Nodes = append(p.Nodes, nodeToRPC(n))
-		if len(p.Nodes) == maxNeighbors || i == len(closest)-1 {
+		if len(p.Nodes) == maxNeighbors {
 			t.send(from, neighborsPacket, &p)
 			p.Nodes = p.Nodes[:0]
+			sent = true
 		}
 	}
+	if len(p.Nodes) > 0 || !sent {
+		t.send(from, neighborsPacket, &p)
+	}
 	return nil
 }
 
diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go
index b81caf8392..db9804f7bc 100644
--- a/p2p/discover/udp_test.go
+++ b/p2p/discover/udp_test.go
@@ -70,14 +70,15 @@ func newUDPTest(t *testing.T) *udpTest {
 		remotekey:  newkey(),
 		remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303},
 	}
-	realaddr := test.pipe.LocalAddr().(*net.UDPAddr)
-	test.table, test.udp, _ = newUDP(test.localkey, test.pipe, realaddr, nil, "", nil)
+	test.table, test.udp, _ = newUDP(test.pipe, Config{PrivateKey: test.localkey})
+	// Wait for initial refresh so the table doesn't send unexpected findnode.
+	<-test.table.initDone
 	return test
 }
 
 // handles a packet as if it had been sent to the transport.
 func (test *udpTest) packetIn(wantError error, ptype byte, data packet) error {
-	enc, err := encodePacket(test.remotekey, ptype, data)
+	enc, _, err := encodePacket(test.remotekey, ptype, data)
 	if err != nil {
 		return test.errorf("packet (%d) encode error: %v", ptype, err)
 	}
@@ -90,19 +91,19 @@ func (test *udpTest) packetIn(wantError error, ptype byte, data packet) error {
 
 // waits for a packet to be sent by the transport.
 // validate should have type func(*udpTest, X) error, where X is a packet type.
-func (test *udpTest) waitPacketOut(validate interface{}) error {
+func (test *udpTest) waitPacketOut(validate interface{}) ([]byte, error) {
 	dgram := test.pipe.waitPacketOut()
-	p, _, _, err := decodePacket(dgram)
+	p, _, hash, err := decodePacket(dgram)
 	if err != nil {
-		return test.errorf("sent packet decode error: %v", err)
+		return hash, test.errorf("sent packet decode error: %v", err)
 	}
 	fn := reflect.ValueOf(validate)
 	exptype := fn.Type().In(0)
 	if reflect.TypeOf(p) != exptype {
-		return test.errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype)
+		return hash, test.errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype)
 	}
 	fn.Call([]reflect.Value{reflect.ValueOf(p)})
-	return nil
+	return hash, nil
 }
 
 func (test *udpTest) errorf(format string, args ...interface{}) error {
@@ -246,12 +247,8 @@ func TestUDP_findnode(t *testing.T) {
 
 	// ensure there's a bond with the test node,
 	// findnode won't be accepted otherwise.
-	test.table.db.updateNode(NewNode(
-		PubkeyID(&test.remotekey.PublicKey),
-		test.remoteaddr.IP,
-		uint16(test.remoteaddr.Port),
-		99,
-	))
+	test.table.db.updateBondTime(PubkeyID(&test.remotekey.PublicKey), time.Now())
+
 	// check that closest neighbors are returned.
 	test.packetIn(nil, findnodePacket, &findnode{Target: testTarget, Expiration: futureExp})
 	expected := test.table.closest(targetHash, bucketSize)
@@ -351,7 +348,7 @@ func TestUDP_successfulPing(t *testing.T) {
 	})
 
 	// remote is unknown, the table pings back.
-	test.waitPacketOut(func(p *ping) error {
+	hash, _ := test.waitPacketOut(func(p *ping) error {
 		if !reflect.DeepEqual(p.From, test.udp.ourEndpoint) {
 			t.Errorf("got ping.From %v, want %v", p.From, test.udp.ourEndpoint)
 		}
@@ -365,7 +362,7 @@ func TestUDP_successfulPing(t *testing.T) {
 		}
 		return nil
 	})
-	test.packetIn(nil, pongPacket, &pong{Expiration: futureExp})
+	test.packetIn(nil, pongPacket, &pong{ReplyTok: hash, Expiration: futureExp})
 
 	// the node should be added to the table shortly after getting the
 	// pong packet.
diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go
index f9baf126f1..52c677b623 100644
--- a/p2p/discv5/net.go
+++ b/p2p/discv5/net.go
@@ -565,11 +565,8 @@ loop:
 			if lookupChn := searchInfo[res.target.topic].lookupChn; lookupChn != nil {
 				lookupChn <- net.ticketStore.radius[res.target.topic].converged
 			}
-			net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node) []byte {
-				net.ping(n, n.addr())
-				return n.pingEcho
-			}, func(n *Node, topic Topic) []byte {
-				if n.state == known {
+			net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte {
+				if n.state != nil && n.state.canQuery {
 					return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration
 				} else {
 					if n.state == unknown {
@@ -633,15 +630,20 @@ loop:
 			}
 			net.refreshResp <- refreshDone
 		case <-refreshDone:
-			log.Trace("<-net.refreshDone")
-			refreshDone = nil
-			list := searchReqWhenRefreshDone
-			searchReqWhenRefreshDone = nil
-			go func() {
-				for _, req := range list {
-					net.topicSearchReq <- req
-				}
-			}()
+			log.Trace("<-net.refreshDone", "table size", net.tab.count)
+			if net.tab.count != 0 {
+				refreshDone = nil
+				list := searchReqWhenRefreshDone
+				searchReqWhenRefreshDone = nil
+				go func() {
+					for _, req := range list {
+						net.topicSearchReq <- req
+					}
+				}()
+			} else {
+				refreshDone = make(chan struct{})
+				net.refresh(refreshDone)
+			}
 		}
 	}
 	log.Trace("loop stopped")
@@ -751,7 +753,15 @@ func (net *Network) internNodeFromNeighbours(sender *net.UDPAddr, rn rpcNode) (n
 		return n, err
 	}
 	if !n.IP.Equal(rn.IP) || n.UDP != rn.UDP || n.TCP != rn.TCP {
-		err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n)
+		if n.state == known {
+			// reject address change if node is known by us
+			err = fmt.Errorf("metadata mismatch: got %v, want %v", rn, n)
+		} else {
+			// accept otherwise; this will be handled nicer with signed ENRs
+			n.IP = rn.IP
+			n.UDP = rn.UDP
+			n.TCP = rn.TCP
+		}
 	}
 	return n, err
 }
diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go
index 37ce8d23cb..b3d1ac4baf 100644
--- a/p2p/discv5/ticket.go
+++ b/p2p/discv5/ticket.go
@@ -494,13 +494,13 @@ func (s *ticketStore) registerLookupDone(lookup lookupInfo, nodes []*Node, ping
 	}
 }
 
-func (s *ticketStore) searchLookupDone(lookup lookupInfo, nodes []*Node, ping func(n *Node) []byte, query func(n *Node, topic Topic) []byte) {
+func (s *ticketStore) searchLookupDone(lookup lookupInfo, nodes []*Node, query func(n *Node, topic Topic) []byte) {
 	now := mclock.Now()
 	for i, n := range nodes {
 		if i == 0 || (binary.BigEndian.Uint64(n.sha[:8])^binary.BigEndian.Uint64(lookup.target[:8])) < s.radius[lookup.topic].minRadius {
 			if lookup.radiusLookup {
 				if lastReq, ok := s.nodeLastReq[n]; !ok || time.Duration(now-lastReq.time) > radiusTC {
-					s.nodeLastReq[n] = reqInfo{pingHash: ping(n), lookup: lookup, time: now}
+					s.nodeLastReq[n] = reqInfo{pingHash: nil, lookup: lookup, time: now}
 				}
 			} // else {
 			if s.canQueryTopic(n, lookup.topic) {
diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go
index 5437718173..6ce72d2c15 100644
--- a/p2p/discv5/udp.go
+++ b/p2p/discv5/udp.go
@@ -49,7 +49,7 @@ var (
 // Timeouts
 const (
 	respTimeout = 500 * time.Millisecond
-	sendTimeout = 500 * time.Millisecond
+	queryDelay  = 1000 * time.Millisecond
 	expiration  = 20 * time.Second
 
 	ntpFailureThreshold = 32               // Continuous timeouts after which to check NTP
@@ -318,20 +318,20 @@ func (t *udp) sendTopicRegister(remote *Node, topics []Topic, idx int, pong []by
 
 func (t *udp) sendTopicNodes(remote *Node, queryHash common.Hash, nodes []*Node) {
 	p := topicNodes{Echo: queryHash}
-	if len(nodes) == 0 {
-		t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p)
-		return
-	}
-	for i, result := range nodes {
-		if netutil.CheckRelayIP(remote.IP, result.IP) != nil {
-			continue
+	var sent bool
+	for _, result := range nodes {
+		if result.IP.Equal(t.net.tab.self.IP) || netutil.CheckRelayIP(remote.IP, result.IP) == nil {
+			p.Nodes = append(p.Nodes, nodeToRPC(result))
 		}
-		p.Nodes = append(p.Nodes, nodeToRPC(result))
-		if len(p.Nodes) == maxTopicNodes || i == len(nodes)-1 {
+		if len(p.Nodes) == maxTopicNodes {
 			t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p)
 			p.Nodes = p.Nodes[:0]
+			sent = true
 		}
 	}
+	if !sent || len(p.Nodes) > 0 {
+		t.sendPacket(remote.ID, remote.addr(), byte(topicNodesPacket), p)
+	}
 }
 
 func (t *udp) sendPacket(toid NodeID, toaddr *net.UDPAddr, ptype byte, req interface{}) (hash []byte, err error) {
diff --git a/p2p/message.go b/p2p/message.go
index 5690494bf4..50b419970d 100644
--- a/p2p/message.go
+++ b/p2p/message.go
@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -112,30 +110,6 @@ func SendItems(w MsgWriter, msgcode uint64, elems ...interface{}) error {
 	return Send(w, msgcode, elems)
 }
 
-// netWrapper wraps a MsgReadWriter with locks around
-// ReadMsg/WriteMsg and applies read/write deadlines.
-type netWrapper struct {
-	rmu, wmu sync.Mutex
-
-	rtimeout, wtimeout time.Duration
-	conn               net.Conn
-	wrapped            MsgReadWriter
-}
-
-func (rw *netWrapper) ReadMsg() (Msg, error) {
-	rw.rmu.Lock()
-	defer rw.rmu.Unlock()
-	rw.conn.SetReadDeadline(time.Now().Add(rw.rtimeout))
-	return rw.wrapped.ReadMsg()
-}
-
-func (rw *netWrapper) WriteMsg(msg Msg) error {
-	rw.wmu.Lock()
-	defer rw.wmu.Unlock()
-	rw.conn.SetWriteDeadline(time.Now().Add(rw.wtimeout))
-	return rw.wrapped.WriteMsg(msg)
-}
-
 // eofSignal wraps a reader with eof signaling. the eof channel is
 // closed when the wrapped reader returns an error or when count bytes
 // have been read.
diff --git a/p2p/metrics.go b/p2p/metrics.go
index 98b61901d5..4cbff90aca 100644
--- a/p2p/metrics.go
+++ b/p2p/metrics.go
@@ -25,10 +25,10 @@ import (
 )
 
 var (
-	ingressConnectMeter = metrics.NewMeter("p2p/InboundConnects")
-	ingressTrafficMeter = metrics.NewMeter("p2p/InboundTraffic")
-	egressConnectMeter  = metrics.NewMeter("p2p/OutboundConnects")
-	egressTrafficMeter  = metrics.NewMeter("p2p/OutboundTraffic")
+	ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil)
+	ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil)
+	egressConnectMeter  = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil)
+	egressTrafficMeter  = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil)
 )
 
 // meteredConn is a wrapper around a network TCP connection that meters both the
diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go
index f6005afd21..656abb6825 100644
--- a/p2p/netutil/net.go
+++ b/p2p/netutil/net.go
@@ -18,8 +18,11 @@
 package netutil
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
 	"net"
+	"sort"
 	"strings"
 )
 
@@ -189,3 +192,131 @@ func CheckRelayIP(sender, addr net.IP) error {
 	}
 	return nil
 }
+
+// SameNet reports whether two IP addresses have an equal prefix of the given bit length.
+func SameNet(bits uint, ip, other net.IP) bool {
+	ip4, other4 := ip.To4(), other.To4()
+	switch {
+	case (ip4 == nil) != (other4 == nil):
+		return false
+	case ip4 != nil:
+		return sameNet(bits, ip4, other4)
+	default:
+		return sameNet(bits, ip.To16(), other.To16())
+	}
+}
+
+func sameNet(bits uint, ip, other net.IP) bool {
+	nb := int(bits / 8)
+	mask := ^byte(0xFF >> (bits % 8))
+	if mask != 0 && nb < len(ip) && ip[nb]&mask != other[nb]&mask {
+		return false
+	}
+	return nb <= len(ip) && bytes.Equal(ip[:nb], other[:nb])
+}
+
+// DistinctNetSet tracks IPs, ensuring that at most N of them
+// fall into the same network range.
+type DistinctNetSet struct {
+	Subnet uint // number of common prefix bits
+	Limit  uint // maximum number of IPs in each subnet
+
+	members map[string]uint
+	buf     net.IP
+}
+
+// Add adds an IP address to the set. It returns false (and doesn't add the IP) if the
+// number of existing IPs in the defined range exceeds the limit.
+func (s *DistinctNetSet) Add(ip net.IP) bool {
+	key := s.key(ip)
+	n := s.members[string(key)]
+	if n < s.Limit {
+		s.members[string(key)] = n + 1
+		return true
+	}
+	return false
+}
+
+// Remove removes an IP from the set.
+func (s *DistinctNetSet) Remove(ip net.IP) {
+	key := s.key(ip)
+	if n, ok := s.members[string(key)]; ok {
+		if n == 1 {
+			delete(s.members, string(key))
+		} else {
+			s.members[string(key)] = n - 1
+		}
+	}
+}
+
+// Contains whether the given IP is contained in the set.
+func (s DistinctNetSet) Contains(ip net.IP) bool {
+	key := s.key(ip)
+	_, ok := s.members[string(key)]
+	return ok
+}
+
+// Len returns the number of tracked IPs.
+func (s DistinctNetSet) Len() int {
+	n := uint(0)
+	for _, i := range s.members {
+		n += i
+	}
+	return int(n)
+}
+
+// key encodes the map key for an address into a temporary buffer.
+//
+// The first byte of key is '4' or '6' to distinguish IPv4/IPv6 address types.
+// The remainder of the key is the IP, truncated to the number of bits.
+func (s *DistinctNetSet) key(ip net.IP) net.IP {
+	// Lazily initialize storage.
+	if s.members == nil {
+		s.members = make(map[string]uint)
+		s.buf = make(net.IP, 17)
+	}
+	// Canonicalize ip and bits.
+	typ := byte('6')
+	if ip4 := ip.To4(); ip4 != nil {
+		typ, ip = '4', ip4
+	}
+	bits := s.Subnet
+	if bits > uint(len(ip)*8) {
+		bits = uint(len(ip) * 8)
+	}
+	// Encode the prefix into s.buf.
+	nb := int(bits / 8)
+	mask := ^byte(0xFF >> (bits % 8))
+	s.buf[0] = typ
+	buf := append(s.buf[:1], ip[:nb]...)
+	if nb < len(ip) && mask != 0 {
+		buf = append(buf, ip[nb]&mask)
+	}
+	return buf
+}
+
+// String implements fmt.Stringer
+func (s DistinctNetSet) String() string {
+	var buf bytes.Buffer
+	buf.WriteString("{")
+	keys := make([]string, 0, len(s.members))
+	for k := range s.members {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		var ip net.IP
+		if k[0] == '4' {
+			ip = make(net.IP, 4)
+		} else {
+			ip = make(net.IP, 16)
+		}
+		copy(ip, k[1:])
+		fmt.Fprintf(&buf, "%v×%d", ip, s.members[k])
+		if i != len(keys)-1 {
+			buf.WriteString(" ")
+		}
+	}
+	buf.WriteString("}")
+	return buf.String()
+}
diff --git a/p2p/netutil/net_test.go b/p2p/netutil/net_test.go
index 1ee1fcb4d6..3a6aa081f2 100644
--- a/p2p/netutil/net_test.go
+++ b/p2p/netutil/net_test.go
@@ -17,9 +17,11 @@
 package netutil
 
 import (
+	"fmt"
 	"net"
 	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/davecgh/go-spew/spew"
 )
@@ -171,3 +173,90 @@ func BenchmarkCheckRelayIP(b *testing.B) {
 		CheckRelayIP(sender, addr)
 	}
 }
+
+func TestSameNet(t *testing.T) {
+	tests := []struct {
+		ip, other string
+		bits      uint
+		want      bool
+	}{
+		{"0.0.0.0", "0.0.0.0", 32, true},
+		{"0.0.0.0", "0.0.0.1", 0, true},
+		{"0.0.0.0", "0.0.0.1", 31, true},
+		{"0.0.0.0", "0.0.0.1", 32, false},
+		{"0.33.0.1", "0.34.0.2", 8, true},
+		{"0.33.0.1", "0.34.0.2", 13, true},
+		{"0.33.0.1", "0.34.0.2", 15, false},
+	}
+
+	for _, test := range tests {
+		if ok := SameNet(test.bits, parseIP(test.ip), parseIP(test.other)); ok != test.want {
+			t.Errorf("SameNet(%d, %s, %s) == %t, want %t", test.bits, test.ip, test.other, ok, test.want)
+		}
+	}
+}
+
+func ExampleSameNet() {
+	// This returns true because the IPs are in the same /24 network:
+	fmt.Println(SameNet(24, net.IP{127, 0, 0, 1}, net.IP{127, 0, 0, 3}))
+	// This call returns false:
+	fmt.Println(SameNet(24, net.IP{127, 3, 0, 1}, net.IP{127, 5, 0, 3}))
+	// Output:
+	// true
+	// false
+}
+
+func TestDistinctNetSet(t *testing.T) {
+	ops := []struct {
+		add, remove string
+		fails       bool
+	}{
+		{add: "127.0.0.1"},
+		{add: "127.0.0.2"},
+		{add: "127.0.0.3", fails: true},
+		{add: "127.32.0.1"},
+		{add: "127.32.0.2"},
+		{add: "127.32.0.3", fails: true},
+		{add: "127.33.0.1", fails: true},
+		{add: "127.34.0.1"},
+		{add: "127.34.0.2"},
+		{add: "127.34.0.3", fails: true},
+		// Make room for an address, then add again.
+		{remove: "127.0.0.1"},
+		{add: "127.0.0.3"},
+		{add: "127.0.0.3", fails: true},
+	}
+
+	set := DistinctNetSet{Subnet: 15, Limit: 2}
+	for _, op := range ops {
+		var desc string
+		if op.add != "" {
+			desc = fmt.Sprintf("Add(%s)", op.add)
+			if ok := set.Add(parseIP(op.add)); ok != !op.fails {
+				t.Errorf("%s == %t, want %t", desc, ok, !op.fails)
+			}
+		} else {
+			desc = fmt.Sprintf("Remove(%s)", op.remove)
+			set.Remove(parseIP(op.remove))
+		}
+		t.Logf("%s: %v", desc, set)
+	}
+}
+
+func TestDistinctNetSetAddRemove(t *testing.T) {
+	cfg := &quick.Config{}
+	fn := func(ips []net.IP) bool {
+		s := DistinctNetSet{Limit: 3, Subnet: 2}
+		for _, ip := range ips {
+			s.Add(ip)
+		}
+		for _, ip := range ips {
+			s.Remove(ip)
+		}
+		return s.Len() == 0
+	}
+
+	if err := quick.Check(fn, cfg); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/p2p/peer.go b/p2p/peer.go
index bad1c8c8b2..477d8c2190 100644
--- a/p2p/peer.go
+++ b/p2p/peer.go
@@ -419,6 +419,9 @@ type PeerInfo struct {
 	Network struct {
 		LocalAddress  string `json:"localAddress"`  // Local endpoint of the TCP data connection
 		RemoteAddress string `json:"remoteAddress"` // Remote endpoint of the TCP data connection
+		Inbound       bool   `json:"inbound"`
+		Trusted       bool   `json:"trusted"`
+		Static        bool   `json:"static"`
 	} `json:"network"`
 	Protocols map[string]interface{} `json:"protocols"` // Sub-protocol specific metadata fields
 }
@@ -439,6 +442,9 @@ func (p *Peer) Info() *PeerInfo {
 	}
 	info.Network.LocalAddress = p.LocalAddr().String()
 	info.Network.RemoteAddress = p.RemoteAddr().String()
+	info.Network.Inbound = p.rw.is(inboundConn)
+	info.Network.Trusted = p.rw.is(trustedConn)
+	info.Network.Static = p.rw.is(staticDialedConn)
 
 	// Gather all the running protocol infos
 	for _, proto := range p.running {
diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go
new file mode 100644
index 0000000000..9914c99587
--- /dev/null
+++ b/p2p/protocols/protocol.go
@@ -0,0 +1,311 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+/*
+Package protocols is an extension to p2p. It offers a user friendly simple way to define
+devp2p subprotocols by abstracting away code standardly shared by protocols.
+
+* automate assigments of code indexes to messages
+* automate RLP decoding/encoding based on reflecting
+* provide the forever loop to read incoming messages
+* standardise error handling related to communication
+* standardised	handshake negotiation
+* TODO: automatic generation of wire protocol specification for peers
+
+*/
+package protocols
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/p2p"
+)
+
+// error codes used by this  protocol scheme
+const (
+	ErrMsgTooLong = iota
+	ErrDecode
+	ErrWrite
+	ErrInvalidMsgCode
+	ErrInvalidMsgType
+	ErrHandshake
+	ErrNoHandler
+	ErrHandler
+)
+
+// error description strings associated with the codes
+var errorToString = map[int]string{
+	ErrMsgTooLong:     "Message too long",
+	ErrDecode:         "Invalid message (RLP error)",
+	ErrWrite:          "Error sending message",
+	ErrInvalidMsgCode: "Invalid message code",
+	ErrInvalidMsgType: "Invalid message type",
+	ErrHandshake:      "Handshake error",
+	ErrNoHandler:      "No handler registered error",
+	ErrHandler:        "Message handler error",
+}
+
+/*
+Error implements the standard go error interface.
+Use:
+
+  errorf(code, format, params ...interface{})
+
+Prints as:
+
+ : 
+ +where description is given by code in errorToString +and details is fmt.Sprintf(format, params...) + +exported field Code can be checked +*/ +type Error struct { + Code int + message string + format string + params []interface{} +} + +func (e Error) Error() (message string) { + if len(e.message) == 0 { + name, ok := errorToString[e.Code] + if !ok { + panic("invalid message code") + } + e.message = name + if e.format != "" { + e.message += ": " + fmt.Sprintf(e.format, e.params...) + } + } + return e.message +} + +func errorf(code int, format string, params ...interface{}) *Error { + return &Error{ + Code: code, + format: format, + params: params, + } +} + +// Spec is a protocol specification including its name and version as well as +// the types of messages which are exchanged +type Spec struct { + // Name is the name of the protocol, often a three-letter word + Name string + + // Version is the version number of the protocol + Version uint + + // MaxMsgSize is the maximum accepted length of the message payload + MaxMsgSize uint32 + + // Messages is a list of message data types which this protocol uses, with + // each message type being sent with its array index as the code (so + // [&foo{}, &bar{}, &baz{}] would send foo, bar and baz with codes + // 0, 1 and 2 respectively) + // each message must have a single unique data type + Messages []interface{} + + initOnce sync.Once + codes map[reflect.Type]uint64 + types map[uint64]reflect.Type +} + +func (s *Spec) init() { + s.initOnce.Do(func() { + s.codes = make(map[reflect.Type]uint64, len(s.Messages)) + s.types = make(map[uint64]reflect.Type, len(s.Messages)) + for i, msg := range s.Messages { + code := uint64(i) + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + s.codes[typ] = code + s.types[code] = typ + } + }) +} + +// Length returns the number of message types in the protocol +func (s *Spec) Length() uint64 { + return uint64(len(s.Messages)) +} + +// GetCode returns the message code of a type, and boolean second argument is +// false if the message type is not found +func (s *Spec) GetCode(msg interface{}) (uint64, bool) { + s.init() + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + code, ok := s.codes[typ] + return code, ok +} + +// NewMsg construct a new message type given the code +func (s *Spec) NewMsg(code uint64) (interface{}, bool) { + s.init() + typ, ok := s.types[code] + if !ok { + return nil, false + } + return reflect.New(typ).Interface(), true +} + +// Peer represents a remote peer or protocol instance that is running on a peer connection with +// a remote peer +type Peer struct { + *p2p.Peer // the p2p.Peer object representing the remote + rw p2p.MsgReadWriter // p2p.MsgReadWriter to send messages to and read messages from + spec *Spec +} + +// NewPeer constructs a new peer +// this constructor is called by the p2p.Protocol#Run function +// the first two arguments are the arguments passed to p2p.Protocol.Run function +// the third argument is the Spec describing the protocol +func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { + return &Peer{ + Peer: p, + rw: rw, + spec: spec, + } +} + +// Run starts the forever loop that handles incoming messages +// called within the p2p.Protocol#Run function +// the handler argument is a function which is called for each message received +// from the remote peer, a returned error causes the loop to exit +// resulting in disconnection +func (p *Peer) Run(handler func(msg interface{}) error) error { + for { + if err := p.handleIncoming(handler); err != nil { + return err + } + } +} + +// Drop disconnects a peer. +// TODO: may need to implement protocol drop only? don't want to kick off the peer +// if they are useful for other protocols +func (p *Peer) Drop(err error) { + p.Disconnect(p2p.DiscSubprotocolError) +} + +// Send takes a message, encodes it in RLP, finds the right message code and sends the +// message off to the peer +// this low level call will be wrapped by libraries providing routed or broadcast sends +// but often just used to forward and push messages to directly connected peers +func (p *Peer) Send(msg interface{}) error { + code, found := p.spec.GetCode(msg) + if !found { + return errorf(ErrInvalidMsgType, "%v", code) + } + return p2p.Send(p.rw, code, msg) +} + +// handleIncoming(code) +// is called each cycle of the main forever loop that dispatches incoming messages +// if this returns an error the loop returns and the peer is disconnected with the error +// this generic handler +// * checks message size, +// * checks for out-of-range message codes, +// * handles decoding with reflection, +// * call handlers as callbacks +func (p *Peer) handleIncoming(handle func(msg interface{}) error) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + // make sure that the payload has been fully consumed + defer msg.Discard() + + if msg.Size > p.spec.MaxMsgSize { + return errorf(ErrMsgTooLong, "%v > %v", msg.Size, p.spec.MaxMsgSize) + } + + val, ok := p.spec.NewMsg(msg.Code) + if !ok { + return errorf(ErrInvalidMsgCode, "%v", msg.Code) + } + if err := msg.Decode(val); err != nil { + return errorf(ErrDecode, "<= %v: %v", msg, err) + } + + // call the registered handler callbacks + // a registered callback take the decoded message as argument as an interface + // which the handler is supposed to cast to the appropriate type + // it is entirely safe not to check the cast in the handler since the handler is + // chosen based on the proper type in the first place + if err := handle(val); err != nil { + return errorf(ErrHandler, "(msg code %v): %v", msg.Code, err) + } + return nil +} + +// Handshake negotiates a handshake on the peer connection +// * arguments +// * context +// * the local handshake to be sent to the remote peer +// * funcion to be called on the remote handshake (can be nil) +// * expects a remote handshake back of the same type +// * the dialing peer needs to send the handshake first and then waits for remote +// * the listening peer waits for the remote handshake and then sends it +// returns the remote handshake and an error +func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interface{}) error) (rhs interface{}, err error) { + if _, ok := p.spec.GetCode(hs); !ok { + return nil, errorf(ErrHandshake, "unknown handshake message type: %T", hs) + } + errc := make(chan error, 2) + handle := func(msg interface{}) error { + rhs = msg + if verify != nil { + return verify(rhs) + } + return nil + } + send := func() { errc <- p.Send(hs) } + receive := func() { errc <- p.handleIncoming(handle) } + + go func() { + if p.Inbound() { + receive() + send() + } else { + send() + receive() + } + }() + + for i := 0; i < 2; i++ { + select { + case err = <-errc: + case <-ctx.Done(): + err = ctx.Err() + } + if err != nil { + return nil, errorf(ErrHandshake, err.Error()) + } + } + return rhs, nil +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go new file mode 100644 index 0000000000..053f537a62 --- /dev/null +++ b/p2p/protocols/protocol_test.go @@ -0,0 +1,389 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package protocols + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +// handshake message type +type hs0 struct { + C uint +} + +// message to kill/drop the peer with nodeID +type kill struct { + C discover.NodeID +} + +// message to drop connection +type drop struct { +} + +/// protoHandshake represents module-independent aspects of the protocol and is +// the first message peers send and receive as part the initial exchange +type protoHandshake struct { + Version uint // local and remote peer should have identical version + NetworkID string // local and remote peer should have identical network id +} + +// checkProtoHandshake verifies local and remote protoHandshakes match +func checkProtoHandshake(testVersion uint, testNetworkID string) func(interface{}) error { + return func(rhs interface{}) error { + remote := rhs.(*protoHandshake) + if remote.NetworkID != testNetworkID { + return fmt.Errorf("%s (!= %s)", remote.NetworkID, testNetworkID) + } + + if remote.Version != testVersion { + return fmt.Errorf("%d (!= %d)", remote.Version, testVersion) + } + return nil + } +} + +// newProtocol sets up a protocol +// the run function here demonstrates a typical protocol using peerPool, handshake +// and messages registered to handlers +func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) error { + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + protoHandshake{}, + hs0{}, + kill{}, + drop{}, + }, + } + return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(p, rw, spec) + + // initiate one-off protohandshake and check validity + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + phs := &protoHandshake{42, "420"} + hsCheck := checkProtoHandshake(phs.Version, phs.NetworkID) + _, err := peer.Handshake(ctx, phs, hsCheck) + if err != nil { + return err + } + + lhs := &hs0{42} + // module handshake demonstrating a simple repeatable exchange of same-type message + hs, err := peer.Handshake(ctx, lhs, nil) + if err != nil { + return err + } + + if rmhs := hs.(*hs0); rmhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rmhs.C, lhs.C) + } + + handle := func(msg interface{}) error { + switch msg := msg.(type) { + + case *protoHandshake: + return errors.New("duplicate handshake") + + case *hs0: + rhs := msg + if rhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rhs.C, lhs.C) + } + lhs.C += rhs.C + return peer.Send(lhs) + + case *kill: + // demonstrates use of peerPool, killing another peer connection as a response to a message + id := msg.C + pp.Get(id).Drop(errors.New("killed")) + return nil + + case *drop: + // for testing we can trigger self induced disconnect upon receiving drop message + return errors.New("dropped") + + default: + return fmt.Errorf("unknown message type: %T", msg) + } + } + + pp.Add(peer) + defer pp.Remove(peer) + return peer.Run(handle) + } +} + +func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester { + conf := adapters.RandomNodeConfig() + return p2ptest.NewProtocolTester(t, conf.ID, 2, newProtocol(pp)) +} + +func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + { + Expects: []p2ptest.Expect{ + { + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: id, + }, + }, + }, + { + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: proto, + Peer: id, + }, + }, + }, + } +} + +func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + // TODO: make this more than one handshake + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, proto)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestProtoHandshakeVersionMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error())) +} + +func TestProtoHandshakeNetworkIDMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "421"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 421 (!= 420)").Error())) +} + +func TestProtoHandshakeSuccess(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "420"}) +} + +func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + { + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &hs0{42}, + Peer: id, + }, + }, + }, + { + Triggers: []p2ptest.Trigger{ + { + Code: 1, + Msg: &hs0{resp}, + Peer: id, + }, + }, + }, + } +} + +func runModuleHandshake(t *testing.T, resp uint, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...); err != nil { + t.Fatal(err) + } + if err := s.TestExchanges(moduleHandshakeExchange(id, resp)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestModuleHandshakeError(t *testing.T) { + runModuleHandshake(t, 43, fmt.Errorf("handshake mismatch remote 43 > local 42")) +} + +func TestModuleHandshakeSuccess(t *testing.T) { + runModuleHandshake(t, 42) +} + +// testing complex interactions over multiple peers, relaying, dropping +func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + { + Label: "primary handshake", + Expects: []p2ptest.Expect{ + { + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + { + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + }, + { + Label: "module handshake", + Triggers: []p2ptest.Trigger{ + { + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + { + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + Expects: []p2ptest.Expect{ + { + Code: 1, + Msg: &hs0{42}, + Peer: a, + }, + { + Code: 1, + Msg: &hs0{42}, + Peer: b, + }, + }, + }, + + {Label: "alternative module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{41}, Peer: a}, + {Code: 1, Msg: &hs0{41}, Peer: b}}}, + {Label: "repeated module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{1}, Peer: a}}}, + {Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{{Code: 1, Msg: &hs0{43}, Peer: a}}}} +} + +func runMultiplePeers(t *testing.T, peer int, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + + if err := s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...); err != nil { + t.Fatal(err) + } + // after some exchanges of messages, we can test state changes + // here this is simply demonstrated by the peerPool + // after the handshake negotiations peers must be added to the pool + // time.Sleep(1) + tick := time.NewTicker(10 * time.Millisecond) + timeout := time.NewTimer(1 * time.Second) +WAIT: + for { + select { + case <-tick.C: + if pp.Has(s.IDs[0]) { + break WAIT + } + case <-timeout.C: + t.Fatal("timeout") + } + } + if !pp.Has(s.IDs[1]) { + t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs) + } + + // peer 0 sends kill request for peer with index + err := s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 2, + Msg: &kill{s.IDs[peer]}, + Peer: s.IDs[0], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // the peer not killed sends a drop request + err = s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + { + Code: 3, + Msg: &drop{}, + Peer: s.IDs[(peer+1)%2], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // check the actual discconnect errors on the individual peers + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } + // test if disconnected peers have been removed from peerPool + if pp.Has(s.IDs[peer]) { + t.Fatalf("peer test-%v not dropped: %v (%v)", peer, pp, s.IDs) + } + +} + +func TestMultiplePeersDropSelf(t *testing.T) { + runMultiplePeers(t, 0, + fmt.Errorf("subprotocol error"), + fmt.Errorf("Message handler error: (msg code 3): dropped"), + ) +} + +func TestMultiplePeersDropOther(t *testing.T) { + runMultiplePeers(t, 1, + fmt.Errorf("Message handler error: (msg code 3): dropped"), + fmt.Errorf("subprotocol error"), + ) +} diff --git a/p2p/rlpx.go b/p2p/rlpx.go index 24037ecc13..1889edac99 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -108,17 +108,19 @@ func (t *rlpx) close(err error) { // Tell the remote end why we're disconnecting if possible. if t.rw != nil { if r, ok := err.(DiscReason); ok && r != DiscNetworkError { - t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout)) - SendItems(t.rw, discMsg, r) + // rlpx tries to send DiscReason to disconnected peer + // if the connection is net.Pipe (in-memory simulation) + // it hangs forever, since net.Pipe does not implement + // a write deadline. Because of this only try to send + // the disconnect reason message if there is no error. + if err := t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout)); err == nil { + SendItems(t.rw, discMsg, r) + } } } t.fd.Close() } -// doEncHandshake runs the protocol handshake using authenticated -// messages. the protocol handshake is the first authenticated message -// and also verifies whether the encryption handshake 'worked' and the -// remote side actually provided the right public key. func (t *rlpx) doProtoHandshake(our *protoHandshake) (their *protoHandshake, err error) { // Writing our handshake happens concurrently, we prefer // returning the handshake read error. If the remote side @@ -169,6 +171,10 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, return &hs, nil } +// doEncHandshake runs the protocol handshake using authenticated +// messages. the protocol handshake is the first authenticated message +// and also verifies whether the encryption handshake 'worked' and the +// remote side actually provided the right public key. func (t *rlpx) doEncHandshake(prv *ecdsa.PrivateKey, dial *discover.Node) (discover.NodeID, error) { var ( sec secrets diff --git a/p2p/rlpx_test.go b/p2p/rlpx_test.go index f4cefa650b..bca4604021 100644 --- a/p2p/rlpx_test.go +++ b/p2p/rlpx_test.go @@ -156,14 +156,18 @@ func TestProtocolHandshake(t *testing.T) { node1 = &discover.Node{ID: discover.PubkeyID(&prv1.PublicKey), IP: net.IP{5, 6, 7, 8}, TCP: 44} hs1 = &protoHandshake{Version: 3, ID: node1.ID, Caps: []Cap{{"c", 1}, {"d", 3}}} - fd0, fd1 = net.Pipe() - wg sync.WaitGroup + wg sync.WaitGroup ) + fd0, fd1, err := tcpPipe() + if err != nil { + t.Fatal(err) + } + wg.Add(2) go func() { defer wg.Done() - defer fd1.Close() + defer fd0.Close() rlpx := newRLPX(fd0) remid, err := rlpx.doEncHandshake(prv0, node1) if err != nil { @@ -597,3 +601,31 @@ func TestHandshakeForwardCompatibility(t *testing.T) { t.Errorf("ingress-mac('foo') mismatch:\ngot %x\nwant %x", fooIngressHash, wantFooIngressHash) } } + +// tcpPipe creates an in process full duplex pipe based on a localhost TCP socket +func tcpPipe() (net.Conn, net.Conn, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, nil, err + } + defer l.Close() + + var aconn net.Conn + aerr := make(chan error, 1) + go func() { + var err error + aconn, err = l.Accept() + aerr <- err + }() + + dconn, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + <-aerr + return nil, nil, err + } + if err := <-aerr; err != nil { + dconn.Close() + return nil, nil, err + } + return aconn, dconn, nil +} diff --git a/p2p/server.go b/p2p/server.go index 2cff94ea5b..c41d1dc156 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -36,15 +36,12 @@ import ( ) const ( - defaultDialTimeout = 15 * time.Second - refreshPeersInterval = 30 * time.Second - staticPeerCheckInterval = 15 * time.Second + defaultDialTimeout = 15 * time.Second - // Maximum number of concurrently handshaking inbound connections. - maxAcceptConns = 50 - - // Maximum number of concurrently dialing outbound connections. - maxActiveDialTasks = 16 + // Connectivity defaults. + maxActiveDialTasks = 16 + defaultMaxPendingPeers = 50 + defaultDialRatio = 3 // Maximum time allowed for reading a complete message. // This is effectively the amount of time a connection can be idle. @@ -70,6 +67,11 @@ type Config struct { // Zero defaults to preset values. MaxPendingPeers int `toml:",omitempty"` + // DialRatio controls the ratio of inbound to dialed connections. + // Example: a DialRatio of 2 allows 1/2 of connections to be dialed. + // Setting DialRatio to zero defaults it to 3. + DialRatio int `toml:",omitempty"` + // NoDiscovery can be used to disable the peer discovery mechanism. // Disabling is useful for protocol debugging (manual topology). NoDiscovery bool @@ -138,7 +140,7 @@ type Config struct { EnableMsgEvents bool // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger + Logger log.Logger `toml:",omitempty"` } // Server manages all peer connections. @@ -427,7 +429,6 @@ func (srv *Server) Start() (err error) { if err != nil { return err } - realaddr = conn.LocalAddr().(*net.UDPAddr) if srv.NAT != nil { if !realaddr.IP.IsLoopback() { @@ -447,11 +448,16 @@ func (srv *Server) Start() (err error) { // node table if !srv.NoDiscovery { - ntab, err := discover.ListenUDP(srv.PrivateKey, conn, realaddr, unhandled, srv.NodeDatabase, srv.NetRestrict) - if err != nil { - return err + cfg := discover.Config{ + PrivateKey: srv.PrivateKey, + AnnounceAddr: realaddr, + NodeDBPath: srv.NodeDatabase, + NetRestrict: srv.NetRestrict, + Bootnodes: srv.BootstrapNodes, + Unhandled: unhandled, } - if err := ntab.SetFallbackNodes(srv.BootstrapNodes); err != nil { + ntab, err := discover.ListenUDP(conn, cfg) + if err != nil { return err } srv.ntab = ntab @@ -476,10 +482,7 @@ func (srv *Server) Start() (err error) { srv.DiscV5 = ntab } - dynPeers := (srv.MaxPeers + 1) / 2 - if srv.NoDiscovery { - dynPeers = 0 - } + dynPeers := srv.maxDialedConns() dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) // handshake @@ -536,6 +539,7 @@ func (srv *Server) run(dialstate dialer) { defer srv.loopWG.Done() var ( peers = make(map[discover.NodeID]*Peer) + inboundCount = 0 trusted = make(map[discover.NodeID]bool, len(srv.TrustedNodes)) taskdone = make(chan task, maxActiveDialTasks) runningTasks []task @@ -621,14 +625,14 @@ running: } // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. select { - case c.cont <- srv.encHandshakeChecks(peers, c): + case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c): case <-srv.quit: break running } case c := <-srv.addpeer: // At this point the connection is past the protocol handshake. // Its capabilities are known and the remote identity is verified. - err := srv.protoHandshakeChecks(peers, c) + err := srv.protoHandshakeChecks(peers, inboundCount, c) if err == nil { // The handshakes are done and it passed all checks. p := newPeer(c, srv.Protocols) @@ -639,8 +643,11 @@ running: } name := truncateName(c.name) srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1) - peers[c.id] = p go srv.runPeer(p) + peers[c.id] = p + if p.Inbound() { + inboundCount++ + } } // The dialer logic relies on the assumption that // dial tasks complete after the peer has been added or @@ -655,6 +662,9 @@ running: d := common.PrettyDuration(mclock.Now() - pd.created) pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err) delete(peers, pd.ID()) + if pd.Inbound() { + inboundCount-- + } } } @@ -681,20 +691,22 @@ running: } } -func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) error { +func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { // Drop connections with no matching protocols. if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 { return DiscUselessPeer } // Repeat the encryption handshake checks because the // peer set might have changed between the handshakes. - return srv.encHandshakeChecks(peers, c) + return srv.encHandshakeChecks(peers, inboundCount, c) } -func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) error { +func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { switch { case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers: return DiscTooManyPeers + case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): + return DiscTooManyPeers case peers[c.id] != nil: return DiscAlreadyConnected case c.id == srv.Self().ID: @@ -704,6 +716,21 @@ func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, c *conn) } } +func (srv *Server) maxInboundConns() int { + return srv.MaxPeers - srv.maxDialedConns() +} + +func (srv *Server) maxDialedConns() int { + if srv.NoDiscovery || srv.NoDial { + return 0 + } + r := srv.DialRatio + if r == 0 { + r = defaultDialRatio + } + return srv.MaxPeers / r +} + type tempError interface { Temporary() bool } @@ -714,10 +741,7 @@ func (srv *Server) listenLoop() { defer srv.loopWG.Done() srv.log.Info("RLPx listener up", "self", srv.makeSelf(srv.listener, srv.ntab)) - // This channel acts as a semaphore limiting - // active inbound connections that are lingering pre-handshake. - // If all slots are taken, no further connections are accepted. - tokens := maxAcceptConns + tokens := defaultMaxPendingPeers if srv.MaxPendingPeers > 0 { tokens = srv.MaxPendingPeers } @@ -758,9 +782,6 @@ func (srv *Server) listenLoop() { fd = newMeteredConn(fd, true) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) - - // Spawn the handler. It will give the slot back when the connection - // has been established. go func() { srv.SetupConn(fd, inboundConn, nil) slots <- struct{}{} diff --git a/p2p/simulations/adapters/state.go b/p2p/simulations/adapters/state.go index 8b1dfef904..0d4ecfb0ff 100644 --- a/p2p/simulations/adapters/state.go +++ b/p2p/simulations/adapters/state.go @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + package adapters type SimStateStore struct { diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go new file mode 100644 index 0000000000..45c6e61425 --- /dev/null +++ b/p2p/testing/peerpool.go @@ -0,0 +1,67 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testing + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" +) + +type TestPeer interface { + ID() discover.NodeID + Drop(error) +} + +// TestPeerPool is an example peerPool to demonstrate registration of peer connections +type TestPeerPool struct { + lock sync.Mutex + peers map[discover.NodeID]TestPeer +} + +func NewTestPeerPool() *TestPeerPool { + return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} +} + +func (self *TestPeerPool) Add(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) + self.peers[p.ID()] = p + +} + +func (self *TestPeerPool) Remove(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + delete(self.peers, p.ID()) +} + +func (self *TestPeerPool) Has(id discover.NodeID) bool { + self.lock.Lock() + defer self.lock.Unlock() + _, ok := self.peers[id] + return ok +} + +func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { + self.lock.Lock() + defer self.lock.Unlock() + return self.peers[id] +} diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go new file mode 100644 index 0000000000..361285f06e --- /dev/null +++ b/p2p/testing/protocolsession.go @@ -0,0 +1,280 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testing + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +var errTimedOut = errors.New("timed out") + +// ProtocolSession is a quasi simulation of a pivot node running +// a service and a number of dummy peers that can send (trigger) or +// receive (expect) messages +type ProtocolSession struct { + Server *p2p.Server + IDs []discover.NodeID + adapter *adapters.SimAdapter + events chan *p2p.PeerEvent +} + +// Exchange is the basic units of protocol tests +// the triggers and expects in the arrays are run immediately and asynchronously +// thus one cannot have multiple expects for the SAME peer with DIFFERENT message types +// because it's unpredictable which expect will receive which message +// (with expect #1 and #2, messages might be sent #2 and #1, and both expects will complain about wrong message code) +// an exchange is defined on a session +type Exchange struct { + Label string + Triggers []Trigger + Expects []Expect + Timeout time.Duration +} + +// Trigger is part of the exchange, incoming message for the pivot node +// sent by a peer +type Trigger struct { + Msg interface{} // type of message to be sent + Code uint64 // code of message is given + Peer discover.NodeID // the peer to send the message to + Timeout time.Duration // timeout duration for the sending +} + +// Expect is part of an exchange, outgoing message from the pivot node +// received by a peer +type Expect struct { + Msg interface{} // type of message to expect + Code uint64 // code of message is now given + Peer discover.NodeID // the peer that expects the message + Timeout time.Duration // timeout duration for receiving +} + +// Disconnect represents a disconnect event, used and checked by TestDisconnected +type Disconnect struct { + Peer discover.NodeID // discconnected peer + Error error // disconnect reason +} + +// trigger sends messages from peers +func (self *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := self.adapter.GetNode(trig.Peer) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", trig.Peer) + } + + errc := make(chan error) + + go func() { + errc <- mockNode.Trigger(&trig) + }() + + t := trig.Timeout + if t == time.Duration(0) { + t = 1000 * time.Millisecond + } + select { + case err := <-errc: + return err + case <-time.After(t): + return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer) + } +} + +// expect checks an expectation of a message sent out by the pivot node +func (self *ProtocolSession) expect(exps []Expect) error { + // construct a map of expectations for each node + peerExpects := make(map[discover.NodeID][]Expect) + for _, exp := range exps { + if exp.Msg == nil { + return errors.New("no message to expect") + } + peerExpects[exp.Peer] = append(peerExpects[exp.Peer], exp) + } + + // construct a map of mockNodes for each node + mockNodes := make(map[discover.NodeID]*mockNode) + for nodeID := range peerExpects { + simNode, ok := self.adapter.GetNode(nodeID) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", nodeID) + } + mockNodes[nodeID] = mockNode + } + + // done chanell cancels all created goroutines when function returns + done := make(chan struct{}) + defer close(done) + // errc catches the first error from + errc := make(chan error) + + wg := &sync.WaitGroup{} + wg.Add(len(mockNodes)) + for nodeID, mockNode := range mockNodes { + nodeID := nodeID + mockNode := mockNode + go func() { + defer wg.Done() + + // Sum all Expect timeouts to give the maximum + // time for all expectations to finish. + // mockNode.Expect checks all received messages against + // a list of expected messages and timeout for each + // of them can not be checked separately. + var t time.Duration + for _, exp := range peerExpects[nodeID] { + if exp.Timeout == time.Duration(0) { + t += 2000 * time.Millisecond + } else { + t += exp.Timeout + } + } + alarm := time.NewTimer(t) + defer alarm.Stop() + + // expectErrc is used to check if error returned + // from mockNode.Expect is not nil and to send it to + // errc only in that case. + // done channel will be closed when function + expectErrc := make(chan error) + go func() { + select { + case expectErrc <- mockNode.Expect(peerExpects[nodeID]...): + case <-done: + case <-alarm.C: + } + }() + + select { + case err := <-expectErrc: + if err != nil { + select { + case errc <- err: + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + } + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + + }() + } + + go func() { + wg.Wait() + // close errc when all goroutines finish to return nill err from errc + close(errc) + }() + + return <-errc +} + +// TestExchanges tests a series of exchanges against the session +func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { + for i, e := range exchanges { + if err := self.testExchange(e); err != nil { + return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) + } + log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) + } + return nil +} + +// testExchange tests a single Exchange. +// Default timeout value is 2 seconds. +func (self *ProtocolSession) testExchange(e Exchange) error { + errc := make(chan error) + done := make(chan struct{}) + defer close(done) + + go func() { + for _, trig := range e.Triggers { + err := self.trigger(trig) + if err != nil { + errc <- err + return + } + } + + select { + case errc <- self.expect(e.Expects): + case <-done: + } + }() + + // time out globally or finish when all expectations satisfied + t := e.Timeout + if t == 0 { + t = 2000 * time.Millisecond + } + alarm := time.NewTimer(t) + select { + case err := <-errc: + return err + case <-alarm.C: + return errTimedOut + } +} + +// TestDisconnected tests the disconnections given as arguments +// the disconnect structs describe what disconnect error is expected on which peer +func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { + expects := make(map[discover.NodeID]error) + for _, disconnect := range disconnects { + expects[disconnect.Peer] = disconnect.Error + } + + timeout := time.After(time.Second) + for len(expects) > 0 { + select { + case event := <-self.events: + if event.Type != p2p.PeerEventTypeDrop { + continue + } + expectErr, ok := expects[event.Peer] + if !ok { + continue + } + + if !(expectErr == nil && event.Error == "" || expectErr != nil && expectErr.Error() == event.Error) { + return fmt.Errorf("unexpected error on peer %v. expected '%v', got '%v'", event.Peer, expectErr, event.Error) + } + delete(expects, event.Peer) + case <-timeout: + return fmt.Errorf("timed out waiting for peers to disconnect") + } + } + return nil +} diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go new file mode 100644 index 0000000000..a797412d60 --- /dev/null +++ b/p2p/testing/protocoltester.go @@ -0,0 +1,269 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +the p2p/testing package provides a unit test scheme to check simple +protocol message exchanges with one pivot node and a number of dummy peers +The pivot test node runs a node.Service, the dummy peers run a mock node +that can be used to send and receive messages +*/ + +package testing + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "strings" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" +) + +// ProtocolTester is the tester environment used for unit testing protocol +// message exchanges. It uses p2p/simulations framework +type ProtocolTester struct { + *ProtocolSession + network *simulations.Network +} + +// NewProtocolTester constructs a new ProtocolTester +// it takes as argument the pivot node id, the number of dummy peers and the +// protocol run function called on a peer connection by the p2p server +func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { + services := adapters.Services{ + "test": func(ctx *adapters.ServiceContext) (node.Service, error) { + return &testNode{run}, nil + }, + "mock": func(ctx *adapters.ServiceContext) (node.Service, error) { + return newMockNode(), nil + }, + } + adapter := adapters.NewSimAdapter(services) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{}) + if _, err := net.NewNodeWithConfig(&adapters.NodeConfig{ + ID: id, + EnableMsgEvents: true, + Services: []string{"test"}, + }); err != nil { + panic(err.Error()) + } + if err := net.Start(id); err != nil { + panic(err.Error()) + } + + node := net.GetNode(id).Node.(*adapters.SimNode) + peers := make([]*adapters.NodeConfig, n) + peerIDs := make([]discover.NodeID, n) + for i := 0; i < n; i++ { + peers[i] = adapters.RandomNodeConfig() + peers[i].Services = []string{"mock"} + peerIDs[i] = peers[i].ID + } + events := make(chan *p2p.PeerEvent, 1000) + node.SubscribeEvents(events) + ps := &ProtocolSession{ + Server: node.Server(), + IDs: peerIDs, + adapter: adapter, + events: events, + } + self := &ProtocolTester{ + ProtocolSession: ps, + network: net, + } + + self.Connect(id, peers...) + + return self +} + +// Stop stops the p2p server +func (self *ProtocolTester) Stop() error { + self.Server.Stop() + return nil +} + +// Connect brings up the remote peer node and connects it using the +// p2p/simulations network connection with the in memory network adapter +func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { + for _, peer := range peers { + log.Trace(fmt.Sprintf("start node %v", peer.ID)) + if _, err := self.network.NewNodeWithConfig(peer); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + if err := self.network.Start(peer.ID); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + log.Trace(fmt.Sprintf("connect to %v", peer.ID)) + if err := self.network.Connect(selfID, peer.ID); err != nil { + panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) + } + } + +} + +// testNode wraps a protocol run function and implements the node.Service +// interface +type testNode struct { + run func(*p2p.Peer, p2p.MsgReadWriter) error +} + +func (t *testNode) Protocols() []p2p.Protocol { + return []p2p.Protocol{{ + Length: 100, + Run: t.run, + }} +} + +func (t *testNode) APIs() []rpc.API { + return nil +} + +func (t *testNode) Start(server *p2p.Server) error { + return nil +} + +func (t *testNode) Stop() error { + return nil +} + +// mockNode is a testNode which doesn't actually run a protocol, instead +// exposing channels so that tests can manually trigger and expect certain +// messages +type mockNode struct { + testNode + + trigger chan *Trigger + expect chan []Expect + err chan error + stop chan struct{} + stopOnce sync.Once +} + +func newMockNode() *mockNode { + mock := &mockNode{ + trigger: make(chan *Trigger), + expect: make(chan []Expect), + err: make(chan error), + stop: make(chan struct{}), + } + mock.testNode.run = mock.Run + return mock +} + +// Run is a protocol run function which just loops waiting for tests to +// instruct it to either trigger or expect a message from the peer +func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + for { + select { + case trig := <-m.trigger: + m.err <- p2p.Send(rw, trig.Code, trig.Msg) + case exps := <-m.expect: + m.err <- expectMsgs(rw, exps) + case <-m.stop: + return nil + } + } +} + +func (m *mockNode) Trigger(trig *Trigger) error { + m.trigger <- trig + return <-m.err +} + +func (m *mockNode) Expect(exp ...Expect) error { + m.expect <- exp + return <-m.err +} + +func (m *mockNode) Stop() error { + m.stopOnce.Do(func() { close(m.stop) }) + return nil +} + +func expectMsgs(rw p2p.MsgReadWriter, exps []Expect) error { + matched := make([]bool, len(exps)) + for { + msg, err := rw.ReadMsg() + if err != nil { + if err == io.EOF { + break + } + return err + } + actualContent, err := ioutil.ReadAll(msg.Payload) + if err != nil { + return err + } + var found bool + for i, exp := range exps { + if exp.Code == msg.Code && bytes.Equal(actualContent, mustEncodeMsg(exp.Msg)) { + if matched[i] { + return fmt.Errorf("message #%d received two times", i) + } + matched[i] = true + found = true + break + } + } + if !found { + expected := make([]string, 0) + for i, exp := range exps { + if matched[i] { + continue + } + expected = append(expected, fmt.Sprintf("code %d payload %x", exp.Code, mustEncodeMsg(exp.Msg))) + } + return fmt.Errorf("unexpected message code %d payload %x, expected %s", msg.Code, actualContent, strings.Join(expected, " or ")) + } + done := true + for _, m := range matched { + if !m { + done = false + break + } + } + if done { + return nil + } + } + for i, m := range matched { + if !m { + return fmt.Errorf("expected message #%d not received", i) + } + } + return nil +} + +// mustEncodeMsg uses rlp to encode a message. +// In case of error it panics. +func mustEncodeMsg(msg interface{}) []byte { + contentEnc, err := rlp.EncodeToBytes(msg) + if err != nil { + panic("content encode error: " + err.Error()) + } + return contentEnc +} diff --git a/params/bootnodes.go b/params/bootnodes.go index f6cbadfc04..c7190ae670 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -33,8 +33,8 @@ var MainnetBootnodes = []string{ // TestnetBootnodes are the enode URLs of the P2P bootstrap nodes running on the // Ropsten test network. var TestnetBootnodes = []string{ - "enode://6ce05930c72abc632c58e2e4324f7c7ea478cec0ed4fa2528982cf34483094e9cbc9216e7aa349691242576d552a2a56aaeae426c5303ded677ce455ba1acd9d@13.84.180.240:30303", // US-TX - "enode://20c9ad97c081d63397d7b685a412227a40e23c8bdc6688c6f37e97cfbc22d2b4d1db1510d8f61e6a8866ad7f0e17c02b14182d37ea7c3c8b9c2683aeb6b733a1@52.169.14.227:30303", // IE + "enode://30b7ab30a01c124a6cceca36863ece12c4f5fa68e3ba9b0b51407ccc002eeed3b3102d20a88f1c1d3c3154e2449317b8ef95090e77b312d5cc39354f86d5d606@52.176.7.10:30303", // US-Azure geth + "enode://865a63255b3bb68023b6bffd5095118fcc13e79dcf014fe4e47e065c350c7cc72af2e53eff895f11ba1bbb6a2b33271c1116ee870f266618eadfc2e78aa7349c@52.176.100.77:30303", // US-Azure parity "enode://6332792c4a00e3e4ee0926ed89e0d27ef985424d97b6a45bf0f23e51f0dcb5e66b875777506458aea7af6f9e4ffb69f43f3778ee73c81ed9d34c51c4b16b0b0f@52.232.243.152:30303", // Parity "enode://94c15d1b9e2fe7ce56e458b9a3b672ef11894ddedd0c6f247e0f1d3487f52b66208fb4aeb8179fce6e3a749ea93ed147c37976d67af557508d199d9594c35f09@192.81.208.223:30303", // @gpip } diff --git a/params/config.go b/params/config.go index 345f6394ab..dc02c7ca37 100644 --- a/params/config.go +++ b/params/config.go @@ -31,46 +31,46 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - ByzantiumBlock: big.NewInt(4370000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(4370000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1700000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1700000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(1035301), - + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(1035301), + ConstantinopleBlock: nil, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -82,16 +82,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -115,7 +115,8 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -152,7 +153,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -161,6 +162,7 @@ func (c *ChainConfig) String() string { c.EIP155Block, c.EIP158Block, c.ByzantiumBlock, + c.ConstantinopleBlock, engine, ) } @@ -191,6 +193,10 @@ func (c *ChainConfig) IsByzantium(num *big.Int) bool { return isForked(c.ByzantiumBlock, num) } +func (c *ChainConfig) IsConstantinople(num *big.Int) bool { + return isForked(c.ConstantinopleBlock, num) +} + // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). // // The returned GasTable's fields shouldn't, under any circumstances, be changed. @@ -251,6 +257,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } + if isForkIncompatible(c.ConstantinopleBlock, newcfg.ConstantinopleBlock, head) { + return newCompatError("Constantinople fork block", c.ConstantinopleBlock, newcfg.ConstantinopleBlock) + } return nil } diff --git a/params/version.go b/params/version.go index 32d4a2e23c..c0437cef02 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release + VersionPatch = 2 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/rpc/http.go b/rpc/http.go index 6717899b53..a46d8c2b39 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -31,6 +31,7 @@ import ( "time" "github.com/rs/cors" + "strings" ) const ( @@ -148,8 +149,11 @@ func (t *httpReadWriteNopCloser) Close() error { // NewHTTPServer creates a new HTTP RPC server around an API provider. // // Deprecated: Server implements http.Handler -func NewHTTPServer(cors []string, srv *Server) *http.Server { - return &http.Server{Handler: newCorsHandler(srv, cors)} +func NewHTTPServer(cors []string, vhosts []string, srv *Server) *http.Server { + // Wrap the CORS-handler within a host-handler + handler := newCorsHandler(srv, cors) + handler = newVHostHandler(vhosts, handler) + return &http.Server{Handler: handler} } // ServeHTTP serves JSON-RPC requests over HTTP. @@ -195,7 +199,6 @@ func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { if len(allowedOrigins) == 0 { return srv } - c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: []string{http.MethodPost, http.MethodGet}, @@ -204,3 +207,50 @@ func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { }) return c.Handler(srv) } + +// virtualHostHandler is a handler which validates the Host-header of incoming requests. +// The virtualHostHandler can prevent DNS rebinding attacks, which do not utilize CORS-headers, +// since they do in-domain requests against the RPC api. Instead, we can see on the Host-header +// which domain was used, and validate that against a whitelist. +type virtualHostHandler struct { + vhosts map[string]struct{} + next http.Handler +} + +// ServeHTTP serves JSON-RPC requests over HTTP, implements http.Handler +func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // if r.Host is not set, we can continue serving since a browser would set the Host header + if r.Host == "" { + h.next.ServeHTTP(w, r) + return + } + host, _, err := net.SplitHostPort(r.Host) + if err != nil { + // Either invalid (too many colons) or no port specified + host = r.Host + } + if ipAddr := net.ParseIP(host); ipAddr != nil { + // It's an IP address, we can serve that + h.next.ServeHTTP(w, r) + return + + } + // Not an ip address, but a hostname. Need to validate + if _, exist := h.vhosts["*"]; exist { + h.next.ServeHTTP(w, r) + return + } + if _, exist := h.vhosts[host]; exist { + h.next.ServeHTTP(w, r) + return + } + http.Error(w, "invalid host specified", http.StatusForbidden) +} + +func newVHostHandler(vhosts []string, next http.Handler) http.Handler { + vhostMap := make(map[string]struct{}) + for _, allowedHost := range vhosts { + vhostMap[strings.ToLower(allowedHost)] = struct{}{} + } + return &virtualHostHandler{vhostMap, next} +} diff --git a/rpc/server.go b/rpc/server.go index 30c288349e..11373b504c 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -421,7 +421,7 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) } } } else { - requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.method, r.method}} + requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.service, r.method}} } continue } diff --git a/rpc/types_test.go b/rpc/types_test.go index 30cef9b22e..68b6d3c54f 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2015 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/swarm/api/api.go b/swarm/api/api.go index 8c4bca2ec0..0cf12fdbed 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net/http" + "path" "regexp" "strings" "sync" @@ -31,15 +32,110 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") +//setup metrics +var ( + apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) + apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) + apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) + apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) + apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) + apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) + apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil) + apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) + apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) + apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) + apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) + apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) + apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) + apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) + apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) +) + type Resolver interface { Resolve(string) (common.Hash, error) } +// NoResolverError is returned by MultiResolver.Resolve if no resolver +// can be found for the address. +type NoResolverError struct { + TLD string +} + +func NewNoResolverError(tld string) *NoResolverError { + return &NoResolverError{TLD: tld} +} + +func (e *NoResolverError) Error() string { + if e.TLD == "" { + return "no ENS resolver" + } + return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) +} + +// MultiResolver is used to resolve URL addresses based on their TLDs. +// Each TLD can have multiple resolvers, and the resoluton from the +// first one in the sequence will be returned. +type MultiResolver struct { + resolvers map[string][]Resolver +} + +// MultiResolverOption sets options for MultiResolver and is used as +// arguments for its constructor. +type MultiResolverOption func(*MultiResolver) + +// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers +// for a specific TLD. If TLD is an empty string, the resolver will be added +// to the list of default resolver, the ones that will be used for resolution +// of addresses which do not have their TLD resolver specified. +func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { + return func(m *MultiResolver) { + m.resolvers[tld] = append(m.resolvers[tld], r) + } +} + +// NewMultiResolver creates a new instance of MultiResolver. +func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { + m = &MultiResolver{ + resolvers: make(map[string][]Resolver), + } + for _, o := range opts { + o(m) + } + return m +} + +// Resolve resolves address by choosing a Resolver by TLD. +// If there are more default Resolvers, or for a specific TLD, +// the Hash from the the first one which does not return error +// will be returned. +func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { + rs := m.resolvers[""] + tld := path.Ext(addr) + if tld != "" { + tld = tld[1:] + rstld, ok := m.resolvers[tld] + if ok { + rs = rstld + } + } + if rs == nil { + return h, NewNoResolverError(tld) + } + for _, r := range rs { + h, err = r.Resolve(addr) + if err == nil { + return + } + } + return +} + /* Api implements webserver/file system related content storage and retrieval on top of the dpa @@ -79,6 +175,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(uri *URI) (storage.Key, error) { + apiResolveCount.Inc(1) log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) // if the URI is immutable, check if the address is a hash @@ -93,6 +190,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // if DNS is not configured, check if the address is a hash if self.dns == nil { if !isHash { + apiResolveFail.Inc(1) return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) } return common.Hex2Bytes(uri.Addr), nil @@ -103,6 +201,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { if err == nil { return resolved[:], nil } else if !isHash { + apiResolveFail.Inc(1) return nil, err } return common.Hex2Bytes(uri.Addr), nil @@ -110,16 +209,19 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (storage.Key, error) { + apiPutCount.Inc(1) r := strings.NewReader(content) wg := &sync.WaitGroup{} key, err := self.dpa.Store(r, int64(len(content)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } wg.Wait() @@ -130,8 +232,10 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { // to resolve basePath to content using dpa retrieve // it returns a section reader, mimeType, status and an error func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + apiGetCount.Inc(1) trie, err := loadManifest(self.dpa, key, nil) if err != nil { + apiGetNotFound.Inc(1) status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -145,6 +249,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe key = common.Hex2Bytes(entry.Hash) status = entry.Status if status == http.StatusMultipleChoices { + apiGetHttp300.Inc(1) return } else { mimeType = entry.ContentType @@ -153,6 +258,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } } else { status = http.StatusNotFound + apiGetNotFound.Inc(1) err = fmt.Errorf("manifest entry for '%s' not found", path) log.Warn(fmt.Sprintf("%v", err)) } @@ -160,9 +266,11 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { + apiModifyCount.Inc(1) quitC := make(chan bool) trie, err := loadManifest(self.dpa, key, quitC) if err != nil { + apiModifyFail.Inc(1) return nil, err } if contentHash != "" { @@ -177,19 +285,23 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) } if err := trie.recalcAndStore(); err != nil { + apiModifyFail.Inc(1) return nil, err } return trie.hash, nil } func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { + apiAddFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -208,16 +320,19 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } fkey, err := mw.AddEntry(bytes.NewReader(content), entry) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -227,13 +342,16 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver } func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) { + apiRmFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiRmFileFail.Inc(1) return "", err } mkey, err := self.Resolve(uri) if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -244,16 +362,19 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiRmFileFail.Inc(1) return "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiRmFileFail.Inc(1) return "", err } newMkey, err := mw.Store() if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -262,6 +383,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin } func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) { + apiAppendFileCount.Inc(1) buffSize := offset + addSize if buffSize < existingSize { @@ -290,10 +412,12 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -304,11 +428,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -322,11 +448,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte fkey, err := mw.AddEntry(io.Reader(combinedReader), entry) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -336,6 +464,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte } func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { + uri, err := Parse("bzz:/" + mhash) if err != nil { return nil, nil, err diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index e673f76c42..4ee26bd8ad 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -237,3 +237,128 @@ func TestAPIResolve(t *testing.T) { }) } } + +func TestMultiResolver(t *testing.T) { + doesntResolve := newTestResolver("") + + ethAddr := "swarm.eth" + ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222" + ethResolve := newTestResolver(ethHash) + + testAddr := "swarm.test" + testHash := "0x1111111111111111111111111111111111111111111111111111111111111111" + testResolve := newTestResolver(testHash) + + tests := []struct { + desc string + r Resolver + addr string + result string + err error + }{ + { + desc: "No resolvers, returns error", + r: NewMultiResolver(), + err: NewNoResolverError(""), + }, + { + desc: "One default resolver, returns resolved address", + r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, first doesn't resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Three TLD resolvers, third resolves, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One TLD resolver doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One defautl and one TLD resolver, all doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + err: errors.New(`DNS name not found: "swarm.eth"`), + }, + { + desc: "Two TLD resolvers, both resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + MultiResolverOptionWithResolver(testResolve, "test"), + ), + addr: testAddr, + result: testHash, + }, + { + desc: "One TLD resolver, no default resolver, returns error for different TLD", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: testAddr, + err: NewNoResolverError("test"), + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + res, err := x.r.Resolve(x.addr) + if err == nil { + if x.err != nil { + t.Fatalf("expected error %q, got result %q", x.err, res.Hex()) + } + if res.Hex() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res.Hex()) + } + } else { + if x.err == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.err.Error() { + t.Fatalf("expected error %q, got %q", x.err, err) + } + } + }) + } +} diff --git a/swarm/api/config.go b/swarm/api/config.go index 140c938ae0..6b224140a4 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -48,7 +48,7 @@ type Config struct { *network.SyncParams Contract common.Address EnsRoot common.Address - EnsApi string + EnsAPIs []string Path string ListenAddr string Port string @@ -75,7 +75,7 @@ func NewDefaultConfig() (self *Config) { ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, Path: node.DefaultDataDir(), - EnsApi: node.DefaultIPCEndpoint("geth"), + EnsAPIs: nil, EnsRoot: ens.TestNetAddress, NetworkId: network.NetworkId, SwapEnabled: false, diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index dbd97182fd..9a65412cf9 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -29,11 +29,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" ) //templateMap holds a mapping of an HTTP error code to a template var templateMap map[int]*template.Template +var caseErrors []CaseError + +//metrics variables +var ( + htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) + jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) +) //parameters needed for formatting the correct HTML page type ErrorParams struct { @@ -44,6 +52,13 @@ type ErrorParams struct { Details template.HTML } +//a custom error case struct that would be used to store validators and +//additional error info to display with client responses. +type CaseError struct { + Validator func(*Request) bool + Msg func(*Request) string +} + //we init the error handling right on boot time, so lookup and http response is fast func init() { initErrHandling() @@ -67,6 +82,29 @@ func initErrHandling() { //assign formatted HTML to the code templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) } + + caseErrors = []CaseError{ + { + Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") }, + Msg: func(r *Request) string { + uriCopy := r.uri + uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x") + return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.
+ Please click here if your browser does not redirect you.`, "/"+uriCopy.String()) + }, + }} +} + +//ValidateCaseErrors is a method that process the request object through certain validators +//that assert if certain conditions are met for further information to log as an error +func ValidateCaseErrors(r *Request) string { + for _, err := range caseErrors { + if err.Validator(r) { + return err.Msg(r) + } + } + + return "" } //ShowMultipeChoices is used when a user requests a resource in a manifest which results @@ -75,10 +113,10 @@ func initErrHandling() { //For example, if the user requests bzz://read and that manifest contains entries //"readme.md" and "readinglist.txt", a HTML page is returned with this two links. //This only applies if the manifest has no default entry -func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) { +func ShowMultipleChoices(w http.ResponseWriter, r *Request, list api.ManifestList) { msg := "" if list.Entries == nil { - ShowError(w, r, "Internal Server Error", http.StatusInternalServerError) + ShowError(w, r, "Could not resolve", http.StatusInternalServerError) return } //make links relative @@ -95,7 +133,7 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //create clickable link for each entry msg += "" + e.Path + "
" } - respond(w, r, &ErrorParams{ + respond(w, &r.Request, &ErrorParams{ Code: http.StatusMultipleChoices, Details: template.HTML(msg), Timestamp: time.Now().Format(time.RFC1123), @@ -108,13 +146,15 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //The function just takes a string message which will be displayed in the error page. //The code is used to evaluate which template will be displayed //(and return the correct HTTP status code) -func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { +func ShowError(w http.ResponseWriter, r *Request, msg string, code int) { + additionalMessage := ValidateCaseErrors(r) if code == http.StatusInternalServerError { log.Error(msg) } - respond(w, r, &ErrorParams{ + respond(w, &r.Request, &ErrorParams{ Code: code, Msg: msg, + Details: template.HTML(additionalMessage), Timestamp: time.Now().Format(time.RFC1123), template: getTemplate(code), }) @@ -132,6 +172,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { //return a HTML page func respondHtml(w http.ResponseWriter, params *ErrorParams) { + htmlCounter.Inc(1) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -140,6 +181,7 @@ func respondHtml(w http.ResponseWriter, params *ErrorParams) { //return JSON func respondJson(w http.ResponseWriter, params *ErrorParams) { + jsonCounter.Inc(1) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(params) } diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go index 2c20ba8f99..cc9b996ba4 100644 --- a/swarm/api/http/error_templates.go +++ b/swarm/api/http/error_templates.go @@ -37,7 +37,7 @@ func GetGenericErrorPage() string { - + + Swarm :: Welcome to Swarm + + + + +
+
+ +
+
+

Welcome to Swarm

+
+
+ + + + +

Enter the hash or ENS of a Swarm-hosted file below:

+ + + +
+
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + + +`[1:])) diff --git a/swarm/fuse/swarmfs_util.go b/swarm/fuse/swarmfs_util.go index d39966c0e3..169b674876 100644 --- a/swarm/fuse/swarmfs_util.go +++ b/swarm/fuse/swarmfs_util.go @@ -47,7 +47,6 @@ func externalUnmount(mountPoint string) error { } func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { - fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true) if err != nil { return err @@ -64,11 +63,9 @@ func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash) return nil - } func removeFileFromSwarm(sf *SwarmFile) error { - mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true) if err != nil { return err @@ -83,7 +80,6 @@ func removeFileFromSwarm(sf *SwarmFile) error { } func removeDirectoryFromSwarm(sd *SwarmDir) error { - if len(sd.directories) == 0 && len(sd.files) == 0 { return nil } @@ -103,11 +99,9 @@ func removeDirectoryFromSwarm(sd *SwarmDir) error { } return nil - } func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error { - fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true) if err != nil { return err @@ -124,5 +118,4 @@ func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, le log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash) return nil - } diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go new file mode 100644 index 0000000000..48b231b219 --- /dev/null +++ b/swarm/metrics/flags.go @@ -0,0 +1,91 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +import ( + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + gethmetrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/influxdb" + "gopkg.in/urfave/cli.v1" +) + +var ( + metricsEnableInfluxDBExportFlag = cli.BoolFlag{ + Name: "metrics.influxdb.export", + Usage: "Enable metrics export/push to an external InfluxDB database", + } + metricsInfluxDBEndpointFlag = cli.StringFlag{ + Name: "metrics.influxdb.endpoint", + Usage: "Metrics InfluxDB endpoint", + Value: "http://127.0.0.1:8086", + } + metricsInfluxDBDatabaseFlag = cli.StringFlag{ + Name: "metrics.influxdb.database", + Usage: "Metrics InfluxDB database", + Value: "metrics", + } + metricsInfluxDBUsernameFlag = cli.StringFlag{ + Name: "metrics.influxdb.username", + Usage: "Metrics InfluxDB username", + Value: "", + } + metricsInfluxDBPasswordFlag = cli.StringFlag{ + Name: "metrics.influxdb.password", + Usage: "Metrics InfluxDB password", + Value: "", + } + // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // It is used so that we can group all nodes and average a measurement across all of them, but also so + // that we can select a specific node and inspect its measurements. + // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key + metricsInfluxDBHostTagFlag = cli.StringFlag{ + Name: "metrics.influxdb.host.tag", + Usage: "Metrics InfluxDB `host` tag attached to all measurements", + Value: "localhost", + } +) + +// Flags holds all command-line flags required for metrics collection. +var Flags = []cli.Flag{ + utils.MetricsEnabledFlag, + metricsEnableInfluxDBExportFlag, + metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, +} + +func Setup(ctx *cli.Context) { + if gethmetrics.Enabled { + log.Info("Enabling swarm metrics collection") + var ( + enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name) + endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + ) + + if enableExport { + log.Info("Enabling swarm metrics export to InfluxDB") + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ + "host": hosttag, + }) + } + } +} diff --git a/swarm/network/depo.go b/swarm/network/depo.go index 17540d2f9f..5ffbf8be11 100644 --- a/swarm/network/depo.go +++ b/swarm/network/depo.go @@ -23,9 +23,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) +//metrics variables +var ( + syncReceiveCount = metrics.NewRegisteredCounter("network.sync.recv.count", nil) + syncReceiveIgnore = metrics.NewRegisteredCounter("network.sync.recv.ignore", nil) + syncSendCount = metrics.NewRegisteredCounter("network.sync.send.count", nil) + syncSendRefused = metrics.NewRegisteredCounter("network.sync.send.refused", nil) + syncSendNotFound = metrics.NewRegisteredCounter("network.sync.send.notfound", nil) +) + // Handler for storage/retrieval related protocol requests // implements the StorageHandler interface used by the bzz protocol type Depo struct { @@ -107,6 +117,7 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key)) // not found in memory cache, ie., a genuine store request // create chunk + syncReceiveCount.Inc(1) chunk = storage.NewChunk(req.Key, nil) case chunk.SData == nil: @@ -116,6 +127,7 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { default: // data is found, store request ignored // this should update access count? + syncReceiveIgnore.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v found locally. ignore.", req)) islocal = true //return @@ -172,11 +184,14 @@ func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) SData: chunk.SData, requestTimeout: req.timeout, // } + syncSendCount.Inc(1) p.syncer.addRequest(sreq, DeliverReq) } else { + syncSendRefused.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log())) } } else { + syncSendNotFound.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log())) } } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 2504a46100..8404ffcc2a 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/swarm/network/kademlia" @@ -39,6 +40,12 @@ import ( // connections and disconnections are reported and relayed // to keep the nodetable uptodate +var ( + peersNumGauge = metrics.NewRegisteredGauge("network.peers.num", nil) + addPeerCounter = metrics.NewRegisteredCounter("network.addpeer.count", nil) + removePeerCounter = metrics.NewRegisteredCounter("network.removepeer.count", nil) +) + type Hive struct { listenAddr func() string callInterval uint64 @@ -192,6 +199,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee func (self *Hive) keepAlive() { alarm := time.NewTicker(time.Duration(self.callInterval)).C for { + peersNumGauge.Update(int64(self.kad.Count())) select { case <-alarm: if self.kad.DBCount() > 0 { @@ -223,6 +231,7 @@ func (self *Hive) Stop() error { // called at the end of a successful protocol handshake func (self *Hive) addPeer(p *peer) error { + addPeerCounter.Inc(1) defer func() { select { case self.more <- true: @@ -247,6 +256,7 @@ func (self *Hive) addPeer(p *peer) error { // called after peer disconnected func (self *Hive) removePeer(p *peer) { + removePeerCounter.Inc(1) log.Debug(fmt.Sprintf("bee %v removed", p)) self.kad.Off(p, saveSync) select { diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 0abc42a19c..b5999b52d4 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -24,6 +24,16 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +//For metrics, we want to count how many times peers are added/removed +//at a certain index. Thus we do that with an array of counters with +//entry for each index +var ( + bucketAddIndexCount []metrics.Counter + bucketRmIndexCount []metrics.Counter ) const ( @@ -88,12 +98,14 @@ type Node interface { // params is KadParams configuration func New(addr Address, params *KadParams) *Kademlia { buckets := make([][]Node, params.MaxProx+1) - return &Kademlia{ + kad := &Kademlia{ addr: addr, KadParams: params, buckets: buckets, db: newKadDb(addr, params), } + kad.initMetricsVariables() + return kad } // accessor for KAD base address @@ -138,6 +150,7 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error // TODO: give priority to peers with active traffic if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation self.buckets[index] = append(bucket, node) + bucketAddIndexCount[index].Inc(1) log.Debug(fmt.Sprintf("add node %v to table", node)) self.setProxLimit(index, true) record.node = node @@ -178,6 +191,7 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { defer self.lock.Unlock() index := self.proximityBin(node.Addr()) + bucketRmIndexCount[index].Inc(1) bucket := self.buckets[index] for i := 0; i < len(bucket); i++ { if node.Addr() == bucket[i].Addr() { @@ -426,3 +440,15 @@ func (self *Kademlia) String() string { rows = append(rows, "=========================================================================") return strings.Join(rows, "\n") } + +//We have to build up the array of counters for each index +func (self *Kademlia) initMetricsVariables() { + //create the arrays + bucketAddIndexCount = make([]metrics.Counter, self.MaxProx+1) + bucketRmIndexCount = make([]metrics.Counter, self.MaxProx+1) + //at each index create a metrics counter + for i := 0; i < (self.KadParams.MaxProx + 1); i++ { + bucketAddIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.add.%d.index", i), nil) + bucketRmIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.rm.%d.index", i), nil) + } +} diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index a418c1dbbd..1cbe00a973 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -39,12 +39,26 @@ import ( "github.com/ethereum/go-ethereum/contracts/chequebook" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" bzzswap "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/services/swap/swap" "github.com/ethereum/go-ethereum/swarm/storage" ) +//metrics variables +var ( + storeRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.storerequest.count", nil) + retrieveRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.retrieverequest.count", nil) + peersMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.peers.count", nil) + syncRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.syncrequest.count", nil) + unsyncedKeysMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.unsyncedkeys.count", nil) + deliverRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.deliverrequest.count", nil) + paymentMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.payment.count", nil) + invalidMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.invalid.count", nil) + handleStatusMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.handlestatus.count", nil) +) + const ( Version = 0 ProtocolLength = uint64(8) @@ -206,6 +220,7 @@ func (self *bzz) handle() error { case storeRequestMsg: // store requests are dispatched to netStore + storeRequestMsgCounter.Inc(1) var req storeRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -221,6 +236,7 @@ func (self *bzz) handle() error { case retrieveRequestMsg: // retrieve Requests are dispatched to netStore + retrieveRequestMsgCounter.Inc(1) var req retrieveRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -241,6 +257,7 @@ func (self *bzz) handle() error { case peersMsg: // response to lookups and immediate response to retrieve requests // dispatches new peer data to the hive that adds them to KADDB + peersMsgCounter.Inc(1) var req peersMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -250,6 +267,7 @@ func (self *bzz) handle() error { self.hive.HandlePeersMsg(&req, &peer{bzz: self}) case syncRequestMsg: + syncRequestMsgCounter.Inc(1) var req syncRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -260,6 +278,7 @@ func (self *bzz) handle() error { case unsyncedKeysMsg: // coming from parent node offering + unsyncedKeysMsgCounter.Inc(1) var req unsyncedKeysMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -274,6 +293,7 @@ func (self *bzz) handle() error { case deliveryRequestMsg: // response to syncKeysMsg hashes filtered not existing in db // also relays the last synced state to the source + deliverRequestMsgCounter.Inc(1) var req deliveryRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<-msg %v: %v", msg, err) @@ -287,6 +307,7 @@ func (self *bzz) handle() error { case paymentMsg: // swap protocol message for payment, Units paid for, Cheque paid with + paymentMsgCounter.Inc(1) if self.swapEnabled { var req paymentMsgData if err := msg.Decode(&req); err != nil { @@ -298,6 +319,7 @@ func (self *bzz) handle() error { default: // no other message is allowed + invalidMsgCounter.Inc(1) return fmt.Errorf("invalid message code: %v", msg.Code) } return nil @@ -332,6 +354,8 @@ func (self *bzz) handleStatus() (err error) { return fmt.Errorf("first msg has code %x (!= %x)", msg.Code, statusMsg) } + handleStatusMsgCounter.Inc(1) + if msg.Size > ProtocolMaxMsgSize { return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize) } diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 98cd6e75ea..2b397f8010 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -23,6 +23,8 @@ import ( "io" "sync" "time" + + "github.com/ethereum/go-ethereum/metrics" ) /* @@ -63,6 +65,11 @@ var ( errOperationTimedOut = errors.New("operation timed out") ) +//metrics variables +var ( + newChunkCounter = metrics.NewRegisteredCounter("storage.chunks.new", nil) +) + type TreeChunker struct { branches int64 hashFunc SwarmHasher @@ -298,6 +305,13 @@ func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan * job.parentWg.Done() if chunkC != nil { + //NOTE: this increases the chunk count even if the local node already has this chunk; + //on file upload the node will increase this counter even if the same file has already been uploaded + //So it should be evaluated whether it is worth keeping this counter + //and/or actually better track when the chunk is Put to the local database + //(which may question the need for disambiguation when a completely new chunk has been created + //and/or a chunk is being put to the local DB; for chunk tracking it may be worth distinguishing + newChunkCounter.Inc(1) chunkC <- newChunk } } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 46a5c16ccc..421bb061d5 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -33,11 +33,18 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" ) +//metrics variables +var ( + gcCounter = metrics.NewRegisteredCounter("storage.db.dbstore.gc.count", nil) + dbStoreDeleteCounter = metrics.NewRegisteredCounter("storage.db.dbstore.rm.count", nil) +) + const ( defaultDbCapacity = 5000000 defaultRadius = 0 // not yet used @@ -255,6 +262,7 @@ func (s *DbStore) collectGarbage(ratio float32) { // actual gc for i := 0; i < gcnt; i++ { if s.gcArray[i].value <= cutval { + gcCounter.Inc(1) s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) } } @@ -383,6 +391,7 @@ func (s *DbStore) delete(idx uint64, idxKey []byte) { batch := new(leveldb.Batch) batch.Delete(idxKey) batch.Delete(getDataKey(idx)) + dbStoreDeleteCounter.Inc(1) s.entryCnt-- batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) s.db.Write(batch) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index b442e6cc54..ece0c86155 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -18,6 +18,13 @@ package storage import ( "encoding/binary" + + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +var ( + dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil) ) // LocalStore is a combination of inmemory db over a disk persisted db @@ -39,6 +46,14 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) { }, nil } +func (self *LocalStore) CacheCounter() uint64 { + return uint64(self.memStore.(*MemStore).Counter()) +} + +func (self *LocalStore) DbCounter() uint64 { + return self.DbStore.(*DbStore).Counter() +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { @@ -48,6 +63,7 @@ func (self *LocalStore) Put(chunk *Chunk) { chunk.wg.Add(1) } go func() { + dbStorePutCounter.Inc(1) self.DbStore.Put(chunk) if chunk.wg != nil { chunk.wg.Done() diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 3cb25ac625..d6be542203 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -23,6 +23,13 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +var ( + memstorePutCounter = metrics.NewRegisteredCounter("storage.db.memstore.put.count", nil) + memstoreRemoveCounter = metrics.NewRegisteredCounter("storage.db.memstore.rm.count", nil) ) const ( @@ -130,6 +137,10 @@ func (s *MemStore) setCapacity(c uint) { s.capacity = c } +func (s *MemStore) Counter() uint { + return s.entryCnt +} + // entry (not its copy) is going to be in MemStore func (s *MemStore) Put(entry *Chunk) { if s.capacity == 0 { @@ -145,6 +156,8 @@ func (s *MemStore) Put(entry *Chunk) { s.accessCnt++ + memstorePutCounter.Inc(1) + node := s.memtree bitpos := uint(0) for node.entry == nil { @@ -289,6 +302,7 @@ func (s *MemStore) removeOldest() { } if node.entry.SData != nil { + memstoreRemoveCounter.Inc(1) node.entry = nil s.entryCnt-- } diff --git a/swarm/swarm.go b/swarm/swarm.go index 3be3660b58..0a120db1fa 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -21,7 +21,11 @@ import ( "context" "crypto/ecdsa" "fmt" + "math/big" "net" + "strings" + "time" + "unicode" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -30,9 +34,11 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -41,6 +47,16 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) +var ( + startTime time.Time + updateGaugesPeriod = 5 * time.Second + startCounter = metrics.NewRegisteredCounter("stack,start", nil) + stopCounter = metrics.NewRegisteredCounter("stack,stop", nil) + uptimeGauge = metrics.NewRegisteredGauge("stack.uptime", nil) + dbSizeGauge = metrics.NewRegisteredGauge("storage.db.chunks.size", nil) + cacheSizeGauge = metrics.NewRegisteredGauge("storage.db.cache.size", nil) +) + // the swarm stack type Swarm struct { config *api.Config // swarm configuration @@ -76,7 +92,7 @@ func (self *Swarm) API() *SwarmAPI { // creates a new swarm service instance // implements node.Service -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -86,10 +102,10 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self = &Swarm{ config: config, - swapEnabled: swapEnabled, + swapEnabled: config.SwapEnabled, backend: backend, privateKey: config.Swap.PrivateKey(), - corsString: cors, + corsString: config.Cors, } log.Debug(fmt.Sprintf("Setting up Swarm service components")) @@ -109,8 +125,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self.hive = network.NewHive( common.HexToHash(self.config.BzzKey), // key to hive (kademlia base address) config.HiveParams, // configuration parameters - swapEnabled, // SWAP enabled - syncEnabled, // syncronisation enabled + config.SwapEnabled, // SWAP enabled + config.SyncEnabled, // syncronisation enabled ) log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive")) @@ -133,18 +149,18 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) - // set up high level api - transactOpts := bind.NewKeyedTransactor(self.privateKey) - - if ensClient == nil { - log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") - } else { - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, ensClient) - if err != nil { - return nil, err + if len(config.EnsAPIs) > 0 { + opts := []api.MultiResolverOption{} + for _, c := range config.EnsAPIs { + tld, endpoint, addr := parseEnsAPIAddress(c) + r, err := newEnsClient(endpoint, addr, config) + if err != nil { + return nil, err + } + opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) } + self.dns = api.NewMultiResolver(opts...) } - log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) self.api = api.NewApi(self.dpa, self.dns) // Manifests for Smart Hosting @@ -156,6 +172,95 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e return self, nil } +// parseEnsAPIAddress parses string according to format +// [tld:][contract-addr@]url and returns ENSClientConfig structure +// with endpoint, contract address and TLD. +func parseEnsAPIAddress(s string) (tld, endpoint string, addr common.Address) { + isAllLetterString := func(s string) bool { + for _, r := range s { + if !unicode.IsLetter(r) { + return false + } + } + return true + } + endpoint = s + if i := strings.Index(endpoint, ":"); i > 0 { + if isAllLetterString(endpoint[:i]) && len(endpoint) > i+2 && endpoint[i+1:i+3] != "//" { + tld = endpoint[:i] + endpoint = endpoint[i+1:] + } + } + if i := strings.Index(endpoint, "@"); i > 0 { + addr = common.HexToAddress(endpoint[:i]) + endpoint = endpoint[i+1:] + } + return +} + +// newEnsClient creates a new ENS client for that is a consumer of +// a ENS API on a specific endpoint. It is used as a helper function +// for creating multiple resolvers in NewSwarm function. +func newEnsClient(endpoint string, addr common.Address, config *api.Config) (*ens.ENS, error) { + log.Info("connecting to ENS API", "url", endpoint) + client, err := rpc.Dial(endpoint) + if err != nil { + return nil, fmt.Errorf("error connecting to ENS API %s: %s", endpoint, err) + } + ensClient := ethclient.NewClient(client) + + ensRoot := config.EnsRoot + if addr != (common.Address{}) { + ensRoot = addr + } else { + a, err := detectEnsAddr(client) + if err == nil { + ensRoot = a + } else { + log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", ensRoot), "err", err) + } + } + transactOpts := bind.NewKeyedTransactor(config.Swap.PrivateKey()) + dns, err := ens.NewENS(transactOpts, ensRoot, ensClient) + if err != nil { + return nil, err + } + log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar %v @ address %v", endpoint, ensRoot.Hex())) + return dns, err +} + +// detectEnsAddr determines the ENS contract address by getting both the +// version and genesis hash using the client and matching them to either +// mainnet or testnet addresses +func detectEnsAddr(client *rpc.Client) (common.Address, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var version string + if err := client.CallContext(ctx, &version, "net_version"); err != nil { + return common.Address{}, err + } + + block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) + if err != nil { + return common.Address{}, err + } + + switch { + + case version == "1" && block.Hash() == params.MainnetGenesisHash: + log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) + return ens.MainNetAddress, nil + + case version == "3" && block.Hash() == params.TestnetGenesisHash: + log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) + return ens.TestNetAddress, nil + + default: + return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) + } +} + /* Start is called when the stack is started * starts the network kademlia hive peer management @@ -168,6 +273,7 @@ Start is called when the stack is started */ // implements the node.Service interface func (self *Swarm) Start(srv *p2p.Server) error { + startTime = time.Now() connectPeer := func(url string) error { node, err := discover.ParseNode(url) if err != nil { @@ -213,9 +319,28 @@ func (self *Swarm) Start(srv *p2p.Server) error { } } + self.periodicallyUpdateGauges() + + startCounter.Inc(1) return nil } +func (self *Swarm) periodicallyUpdateGauges() { + ticker := time.NewTicker(updateGaugesPeriod) + + go func() { + for range ticker.C { + self.updateGauges() + } + }() +} + +func (self *Swarm) updateGauges() { + dbSizeGauge.Update(int64(self.lstore.DbCounter())) + cacheSizeGauge.Update(int64(self.lstore.CacheCounter())) + uptimeGauge.Update(time.Since(startTime).Nanoseconds()) +} + // implements the node.Service interface // stops all component services. func (self *Swarm) Stop() error { @@ -230,6 +355,7 @@ func (self *Swarm) Stop() error { self.lstore.DbStore.Close() } self.sfs.Stop() + stopCounter.Inc(1) return err } diff --git a/swarm/swarm_test.go b/swarm/swarm_test.go new file mode 100644 index 0000000000..8b1ae28884 --- /dev/null +++ b/swarm/swarm_test.go @@ -0,0 +1,119 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swarm + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestParseEnsAPIAddress(t *testing.T) { + for _, x := range []struct { + description string + value string + tld string + endpoint string + addr common.Address + }{ + { + description: "IPC endpoint", + value: "/data/testnet/geth.ipc", + endpoint: "/data/testnet/geth.ipc", + }, + { + description: "HTTP endpoint", + value: "http://127.0.0.1:1234", + endpoint: "http://127.0.0.1:1234", + }, + { + description: "WS endpoint", + value: "ws://127.0.0.1:1234", + endpoint: "ws://127.0.0.1:1234", + }, + { + description: "IPC Endpoint and TLD", + value: "test:/data/testnet/geth.ipc", + endpoint: "/data/testnet/geth.ipc", + tld: "test", + }, + { + description: "HTTP endpoint and TLD", + value: "test:http://127.0.0.1:1234", + endpoint: "http://127.0.0.1:1234", + tld: "test", + }, + { + description: "WS endpoint and TLD", + value: "test:ws://127.0.0.1:1234", + endpoint: "ws://127.0.0.1:1234", + tld: "test", + }, + { + description: "IPC Endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + endpoint: "/data/testnet/geth.ipc", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + }, + { + description: "HTTP endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + endpoint: "http://127.0.0.1:1234", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + }, + { + description: "WS endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + endpoint: "ws://127.0.0.1:1234", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + }, + { + description: "IPC Endpoint, TLD and contract address", + value: "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + endpoint: "/data/testnet/geth.ipc", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + tld: "test", + }, + { + description: "HTTP endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + endpoint: "http://127.0.0.1:1234", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + tld: "eth", + }, + { + description: "WS endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + endpoint: "ws://127.0.0.1:1234", + addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"), + tld: "eth", + }, + } { + t.Run(x.description, func(t *testing.T) { + tld, endpoint, addr := parseEnsAPIAddress(x.value) + if endpoint != x.endpoint { + t.Errorf("expected Endpoint %q, got %q", x.endpoint, endpoint) + } + if addr != x.addr { + t.Errorf("expected ContractAddress %q, got %q", x.addr.String(), addr.String()) + } + if tld != x.tld { + t.Errorf("expected TLD %q, got %q", x.tld, tld) + } + }) + } +} diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index a449b1cfa6..6006373009 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -13,7 +13,6 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// package tests diff --git a/tests/difficulty_test_util.go b/tests/difficulty_test_util.go index 7541477933..00d699cf75 100644 --- a/tests/difficulty_test_util.go +++ b/tests/difficulty_test_util.go @@ -13,7 +13,6 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// package tests diff --git a/tests/init.go b/tests/init.go index 9e884efe39..ff8ee7da18 100644 --- a/tests/init.go +++ b/tests/init.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2015 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/tests/init_test.go b/tests/init_test.go index ebb0d32c34..fbb214b08c 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The go-ethereum Authors +// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/tests/state_test.go b/tests/state_test.go index 100c776c1a..9ca5f18303 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2015 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 18280d2a46..3b761bd771 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2015 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/trie/database.go b/trie/database.go index d79120813d..da36e72f98 100644 --- a/trie/database.go +++ b/trie/database.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify diff --git a/trie/trie.go b/trie/trie.go index e37a1ae109..31a404e3a0 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" ) var ( diff --git a/vendor/github.com/influxdata/influxdb/LICENSE b/vendor/github.com/influxdata/influxdb/LICENSE new file mode 100644 index 0000000000..63cef79ba6 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md b/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md new file mode 100644 index 0000000000..ea6fc69f30 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md @@ -0,0 +1,62 @@ +- # List +- bootstrap 3.3.5 [MIT LICENSE](https://github.com/twbs/bootstrap/blob/master/LICENSE) +- collectd.org [ISC LICENSE](https://github.com/collectd/go-collectd/blob/master/LICENSE) +- github.com/BurntSushi/toml [MIT LICENSE](https://github.com/BurntSushi/toml/blob/master/COPYING) +- github.com/RoaringBitmap/roaring [APACHE LICENSE](https://github.com/RoaringBitmap/roaring/blob/master/LICENSE) +- github.com/beorn7/perks [MIT LICENSE](https://github.com/beorn7/perks/blob/master/LICENSE) +- github.com/bmizerany/pat [MIT LICENSE](https://github.com/bmizerany/pat#license) +- github.com/boltdb/bolt [MIT LICENSE](https://github.com/boltdb/bolt/blob/master/LICENSE) +- github.com/cespare/xxhash [MIT LICENSE](https://github.com/cespare/xxhash/blob/master/LICENSE.txt) +- github.com/clarkduvall/hyperloglog [MIT LICENSE](https://github.com/clarkduvall/hyperloglog/blob/master/LICENSE) +- github.com/davecgh/go-spew/spew [ISC LICENSE](https://github.com/davecgh/go-spew/blob/master/LICENSE) +- github.com/dgrijalva/jwt-go [MIT LICENSE](https://github.com/dgrijalva/jwt-go/blob/master/LICENSE) +- github.com/dgryski/go-bits [MIT LICENSE](https://github.com/dgryski/go-bits/blob/master/LICENSE) +- github.com/dgryski/go-bitstream [MIT LICENSE](https://github.com/dgryski/go-bitstream/blob/master/LICENSE) +- github.com/glycerine/go-unsnap-stream [MIT LICENSE](https://github.com/glycerine/go-unsnap-stream/blob/master/LICENSE) +- github.com/gogo/protobuf/proto [BSD LICENSE](https://github.com/gogo/protobuf/blob/master/LICENSE) +- github.com/golang/protobuf [BSD LICENSE](https://github.com/golang/protobuf/blob/master/LICENSE) +- github.com/golang/snappy [BSD LICENSE](https://github.com/golang/snappy/blob/master/LICENSE) +- github.com/google/go-cmp [BSD LICENSE](https://github.com/google/go-cmp/blob/master/LICENSE) +- github.com/influxdata/influxql [MIT LICENSE](https://github.com/influxdata/influxql/blob/master/LICENSE) +- github.com/influxdata/usage-client [MIT LICENSE](https://github.com/influxdata/usage-client/blob/master/LICENSE.txt) +- github.com/influxdata/yamux [MOZILLA PUBLIC LICENSE](https://github.com/influxdata/yamux/blob/master/LICENSE) +- github.com/influxdata/yarpc [MIT LICENSE](https://github.com/influxdata/yarpc/blob/master/LICENSE) +- github.com/jsternberg/zap-logfmt [MIT LICENSE](https://github.com/jsternberg/zap-logfmt/blob/master/LICENSE) +- github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE) +- github.com/mattn/go-isatty [MIT LICENSE](https://github.com/mattn/go-isatty/blob/master/LICENSE) +- github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE) +- github.com/opentracing/opentracing-go [MIT LICENSE](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) +- github.com/paulbellamy/ratecounter [MIT LICENSE](https://github.com/paulbellamy/ratecounter/blob/master/LICENSE) +- github.com/peterh/liner [MIT LICENSE](https://github.com/peterh/liner/blob/master/COPYING) +- github.com/philhofer/fwd [MIT LICENSE](https://github.com/philhofer/fwd/blob/master/LICENSE.md) +- github.com/prometheus/client_golang [MIT LICENSE](https://github.com/prometheus/client_golang/blob/master/LICENSE) +- github.com/prometheus/client_model [MIT LICENSE](https://github.com/prometheus/client_model/blob/master/LICENSE) +- github.com/prometheus/common [APACHE LICENSE](https://github.com/prometheus/common/blob/master/LICENSE) +- github.com/prometheus/procfs [APACHE LICENSE](https://github.com/prometheus/procfs/blob/master/LICENSE) +- github.com/rakyll/statik [APACHE LICENSE](https://github.com/rakyll/statik/blob/master/LICENSE) +- github.com/retailnext/hllpp [BSD LICENSE](https://github.com/retailnext/hllpp/blob/master/LICENSE) +- github.com/tinylib/msgp [MIT LICENSE](https://github.com/tinylib/msgp/blob/master/LICENSE) +- go.uber.org/atomic [MIT LICENSE](https://github.com/uber-go/atomic/blob/master/LICENSE.txt) +- go.uber.org/multierr [MIT LICENSE](https://github.com/uber-go/multierr/blob/master/LICENSE.txt) +- go.uber.org/zap [MIT LICENSE](https://github.com/uber-go/zap/blob/master/LICENSE.txt) +- golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE) +- golang.org/x/net [BSD LICENSE](https://github.com/golang/net/blob/master/LICENSE) +- golang.org/x/sys [BSD LICENSE](https://github.com/golang/sys/blob/master/LICENSE) +- golang.org/x/text [BSD LICENSE](https://github.com/golang/text/blob/master/LICENSE) +- golang.org/x/time [BSD LICENSE](https://github.com/golang/time/blob/master/LICENSE) +- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt) +- github.com/xlab/treeprint [MIT LICENSE](https://github.com/xlab/treeprint/blob/master/LICENSE) + + + + + + + + + + + + + + diff --git a/vendor/github.com/influxdata/influxdb/client/README.md b/vendor/github.com/influxdata/influxdb/client/README.md new file mode 100644 index 0000000000..773a111228 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/README.md @@ -0,0 +1,306 @@ +# InfluxDB Client + +[![GoDoc](https://godoc.org/github.com/influxdata/influxdb?status.svg)](http://godoc.org/github.com/influxdata/influxdb/client/v2) + +## Description + +**NOTE:** The Go client library now has a "v2" version, with the old version +being deprecated. The new version can be imported at +`import "github.com/influxdata/influxdb/client/v2"`. It is not backwards-compatible. + +A Go client library written and maintained by the **InfluxDB** team. +This package provides convenience functions to read and write time series data. +It uses the HTTP protocol to communicate with your **InfluxDB** cluster. + + +## Getting Started + +### Connecting To Your Database + +Connecting to an **InfluxDB** database is straightforward. You will need a host +name, a port and the cluster user credentials if applicable. The default port is +8086. You can customize these settings to your specific installation via the +**InfluxDB** configuration file. + +Though not necessary for experimentation, you may want to create a new user +and authenticate the connection to your database. + +For more information please check out the +[Admin Docs](https://docs.influxdata.com/influxdb/latest/administration/). + +For the impatient, you can create a new admin user _bubba_ by firing off the +[InfluxDB CLI](https://github.com/influxdata/influxdb/blob/master/cmd/influx/main.go). + +```shell +influx +> create user bubba with password 'bumblebeetuna' +> grant all privileges to bubba +``` + +And now for good measure set the credentials in you shell environment. +In the example below we will use $INFLUX_USER and $INFLUX_PWD + +Now with the administrivia out of the way, let's connect to our database. + +NOTE: If you've opted out of creating a user, you can omit Username and Password in +the configuration below. + +```go +package main + +import ( + "log" + "time" + + "github.com/influxdata/influxdb/client/v2" +) + +const ( + MyDB = "square_holes" + username = "bubba" + password = "bumblebeetuna" +) + + +func main() { + // Create a new HTTPClient + c, err := client.NewHTTPClient(client.HTTPConfig{ + Addr: "http://localhost:8086", + Username: username, + Password: password, + }) + if err != nil { + log.Fatal(err) + } + + // Create a new point batch + bp, err := client.NewBatchPoints(client.BatchPointsConfig{ + Database: MyDB, + Precision: "s", + }) + if err != nil { + log.Fatal(err) + } + + // Create a point and add to batch + tags := map[string]string{"cpu": "cpu-total"} + fields := map[string]interface{}{ + "idle": 10.1, + "system": 53.3, + "user": 46.6, + } + + pt, err := client.NewPoint("cpu_usage", tags, fields, time.Now()) + if err != nil { + log.Fatal(err) + } + bp.AddPoint(pt) + + // Write the batch + if err := c.Write(bp); err != nil { + log.Fatal(err) + } +} + +``` + +### Inserting Data + +Time series data aka *points* are written to the database using batch inserts. +The mechanism is to create one or more points and then create a batch aka +*batch points* and write these to a given database and series. A series is a +combination of a measurement (time/values) and a set of tags. + +In this sample we will create a batch of a 1,000 points. Each point has a time and +a single value as well as 2 tags indicating a shape and color. We write these points +to a database called _square_holes_ using a measurement named _shapes_. + +NOTE: You can specify a RetentionPolicy as part of the batch points. If not +provided InfluxDB will use the database _default_ retention policy. + +```go + +func writePoints(clnt client.Client) { + sampleSize := 1000 + + bp, err := client.NewBatchPoints(client.BatchPointsConfig{ + Database: "systemstats", + Precision: "us", + }) + if err != nil { + log.Fatal(err) + } + + rand.Seed(time.Now().UnixNano()) + for i := 0; i < sampleSize; i++ { + regions := []string{"us-west1", "us-west2", "us-west3", "us-east1"} + tags := map[string]string{ + "cpu": "cpu-total", + "host": fmt.Sprintf("host%d", rand.Intn(1000)), + "region": regions[rand.Intn(len(regions))], + } + + idle := rand.Float64() * 100.0 + fields := map[string]interface{}{ + "idle": idle, + "busy": 100.0 - idle, + } + + pt, err := client.NewPoint( + "cpu_usage", + tags, + fields, + time.Now(), + ) + if err != nil { + log.Fatal(err) + } + bp.AddPoint(pt) + } + + if err := clnt.Write(bp); err != nil { + log.Fatal(err) + } +} +``` + +#### Uint64 Support + +The `uint64` data type is supported if your server is version `1.4.0` or +greater. To write a data point as an unsigned integer, you must insert +the point as `uint64`. You cannot use `uint` or any of the other +derivatives because previous versions of the client have supported +writing those types as an integer. + +### Querying Data + +One nice advantage of using **InfluxDB** the ability to query your data using familiar +SQL constructs. In this example we can create a convenience function to query the database +as follows: + +```go +// queryDB convenience function to query the database +func queryDB(clnt client.Client, cmd string) (res []client.Result, err error) { + q := client.Query{ + Command: cmd, + Database: MyDB, + } + if response, err := clnt.Query(q); err == nil { + if response.Error() != nil { + return res, response.Error() + } + res = response.Results + } else { + return res, err + } + return res, nil +} +``` + +#### Creating a Database + +```go +_, err := queryDB(clnt, fmt.Sprintf("CREATE DATABASE %s", MyDB)) +if err != nil { + log.Fatal(err) +} +``` + +#### Count Records + +```go +q := fmt.Sprintf("SELECT count(%s) FROM %s", "value", MyMeasurement) +res, err := queryDB(clnt, q) +if err != nil { + log.Fatal(err) +} +count := res[0].Series[0].Values[0][1] +log.Printf("Found a total of %v records\n", count) +``` + +#### Find the last 10 _shapes_ records + +```go +q := fmt.Sprintf("SELECT * FROM %s LIMIT %d", MyMeasurement, 10) +res, err = queryDB(clnt, q) +if err != nil { + log.Fatal(err) +} + +for i, row := range res[0].Series[0].Values { + t, err := time.Parse(time.RFC3339, row[0].(string)) + if err != nil { + log.Fatal(err) + } + val := row[1].(string) + log.Printf("[%2d] %s: %s\n", i, t.Format(time.Stamp), val) +} +``` + +### Using the UDP Client + +The **InfluxDB** client also supports writing over UDP. + +```go +func WriteUDP() { + // Make client + c, err := client.NewUDPClient("localhost:8089") + if err != nil { + panic(err.Error()) + } + + // Create a new point batch + bp, _ := client.NewBatchPoints(client.BatchPointsConfig{ + Precision: "s", + }) + + // Create a point and add to batch + tags := map[string]string{"cpu": "cpu-total"} + fields := map[string]interface{}{ + "idle": 10.1, + "system": 53.3, + "user": 46.6, + } + pt, err := client.NewPoint("cpu_usage", tags, fields, time.Now()) + if err != nil { + panic(err.Error()) + } + bp.AddPoint(pt) + + // Write the batch + c.Write(bp) +} +``` + +### Point Splitting + +The UDP client now supports splitting single points that exceed the configured +payload size. The logic for processing each point is listed here, starting with +an empty payload. + +1. If adding the point to the current (non-empty) payload would exceed the + configured size, send the current payload. Otherwise, add it to the current + payload. +1. If the point is smaller than the configured size, add it to the payload. +1. If the point has no timestamp, just try to send the entire point as a single + UDP payload, and process the next point. +1. Since the point has a timestamp, re-use the existing measurement name, + tagset, and timestamp and create multiple new points by splitting up the + fields. The per-point length will be kept close to the configured size, + staying under it if possible. This does mean that one large field, maybe a + long string, could be sent as a larger-than-configured payload. + +The above logic attempts to respect configured payload sizes, but not sacrifice +any data integrity. Points without a timestamp can't be split, as that may +cause fields to have differing timestamps when processed by the server. + +## Go Docs + +Please refer to +[http://godoc.org/github.com/influxdata/influxdb/client/v2](http://godoc.org/github.com/influxdata/influxdb/client/v2) +for documentation. + +## See Also + +You can also examine how the client library is used by the +[InfluxDB CLI](https://github.com/influxdata/influxdb/blob/master/cmd/influx/main.go). diff --git a/vendor/github.com/influxdata/influxdb/client/influxdb.go b/vendor/github.com/influxdata/influxdb/client/influxdb.go new file mode 100644 index 0000000000..98d362d50a --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/influxdb.go @@ -0,0 +1,840 @@ +// Package client implements a now-deprecated client for InfluxDB; +// use github.com/influxdata/influxdb/client/v2 instead. +package client // import "github.com/influxdata/influxdb/client" + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/models" +) + +const ( + // DefaultHost is the default host used to connect to an InfluxDB instance + DefaultHost = "localhost" + + // DefaultPort is the default port used to connect to an InfluxDB instance + DefaultPort = 8086 + + // DefaultTimeout is the default connection timeout used to connect to an InfluxDB instance + DefaultTimeout = 0 +) + +// Query is used to send a command to the server. Both Command and Database are required. +type Query struct { + Command string + Database string + + // Chunked tells the server to send back chunked responses. This places + // less load on the server by sending back chunks of the response rather + // than waiting for the entire response all at once. + Chunked bool + + // ChunkSize sets the maximum number of rows that will be returned per + // chunk. Chunks are either divided based on their series or if they hit + // the chunk size limit. + // + // Chunked must be set to true for this option to be used. + ChunkSize int +} + +// ParseConnectionString will parse a string to create a valid connection URL +func ParseConnectionString(path string, ssl bool) (url.URL, error) { + var host string + var port int + + h, p, err := net.SplitHostPort(path) + if err != nil { + if path == "" { + host = DefaultHost + } else { + host = path + } + // If they didn't specify a port, always use the default port + port = DefaultPort + } else { + host = h + port, err = strconv.Atoi(p) + if err != nil { + return url.URL{}, fmt.Errorf("invalid port number %q: %s\n", path, err) + } + } + + u := url.URL{ + Scheme: "http", + } + if ssl { + u.Scheme = "https" + } + + u.Host = net.JoinHostPort(host, strconv.Itoa(port)) + + return u, nil +} + +// Config is used to specify what server to connect to. +// URL: The URL of the server connecting to. +// Username/Password are optional. They will be passed via basic auth if provided. +// UserAgent: If not provided, will default "InfluxDBClient", +// Timeout: If not provided, will default to 0 (no timeout) +type Config struct { + URL url.URL + UnixSocket string + Username string + Password string + UserAgent string + Timeout time.Duration + Precision string + WriteConsistency string + UnsafeSsl bool +} + +// NewConfig will create a config to be used in connecting to the client +func NewConfig() Config { + return Config{ + Timeout: DefaultTimeout, + } +} + +// Client is used to make calls to the server. +type Client struct { + url url.URL + unixSocket string + username string + password string + httpClient *http.Client + userAgent string + precision string +} + +const ( + // ConsistencyOne requires at least one data node acknowledged a write. + ConsistencyOne = "one" + + // ConsistencyAll requires all data nodes to acknowledge a write. + ConsistencyAll = "all" + + // ConsistencyQuorum requires a quorum of data nodes to acknowledge a write. + ConsistencyQuorum = "quorum" + + // ConsistencyAny allows for hinted hand off, potentially no write happened yet. + ConsistencyAny = "any" +) + +// NewClient will instantiate and return a connected client to issue commands to the server. +func NewClient(c Config) (*Client, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: c.UnsafeSsl, + } + + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + } + + if c.UnixSocket != "" { + // No need for compression in local communications. + tr.DisableCompression = true + + tr.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", c.UnixSocket) + } + } + + client := Client{ + url: c.URL, + unixSocket: c.UnixSocket, + username: c.Username, + password: c.Password, + httpClient: &http.Client{Timeout: c.Timeout, Transport: tr}, + userAgent: c.UserAgent, + precision: c.Precision, + } + if client.userAgent == "" { + client.userAgent = "InfluxDBClient" + } + return &client, nil +} + +// SetAuth will update the username and passwords +func (c *Client) SetAuth(u, p string) { + c.username = u + c.password = p +} + +// SetPrecision will update the precision +func (c *Client) SetPrecision(precision string) { + c.precision = precision +} + +// Query sends a command to the server and returns the Response +func (c *Client) Query(q Query) (*Response, error) { + return c.QueryContext(context.Background(), q) +} + +// QueryContext sends a command to the server and returns the Response +// It uses a context that can be cancelled by the command line client +func (c *Client) QueryContext(ctx context.Context, q Query) (*Response, error) { + u := c.url + + u.Path = "query" + values := u.Query() + values.Set("q", q.Command) + values.Set("db", q.Database) + if q.Chunked { + values.Set("chunked", "true") + if q.ChunkSize > 0 { + values.Set("chunk_size", strconv.Itoa(q.ChunkSize)) + } + } + if c.precision != "" { + values.Set("epoch", c.precision) + } + u.RawQuery = values.Encode() + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + req = req.WithContext(ctx) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + if q.Chunked { + cr := NewChunkedResponse(resp.Body) + for { + r, err := cr.NextResponse() + if err != nil { + // If we got an error while decoding the response, send that back. + return nil, err + } + + if r == nil { + break + } + + response.Results = append(response.Results, r.Results...) + if r.Err != nil { + response.Err = r.Err + break + } + } + } else { + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + if err := dec.Decode(&response); err != nil { + // Ignore EOF errors if we got an invalid status code. + if !(err == io.EOF && resp.StatusCode != http.StatusOK) { + return nil, err + } + } + } + + // If we don't have an error in our json response, and didn't get StatusOK, + // then send back an error. + if resp.StatusCode != http.StatusOK && response.Error() == nil { + return &response, fmt.Errorf("received status code %d from server", resp.StatusCode) + } + return &response, nil +} + +// Write takes BatchPoints and allows for writing of multiple points with defaults +// If successful, error is nil and Response is nil +// If an error occurs, Response may contain additional information if populated. +func (c *Client) Write(bp BatchPoints) (*Response, error) { + u := c.url + u.Path = "write" + + var b bytes.Buffer + for _, p := range bp.Points { + err := checkPointTypes(p) + if err != nil { + return nil, err + } + if p.Raw != "" { + if _, err := b.WriteString(p.Raw); err != nil { + return nil, err + } + } else { + for k, v := range bp.Tags { + if p.Tags == nil { + p.Tags = make(map[string]string, len(bp.Tags)) + } + p.Tags[k] = v + } + + if _, err := b.WriteString(p.MarshalString()); err != nil { + return nil, err + } + } + + if err := b.WriteByte('\n'); err != nil { + return nil, err + } + } + + req, err := http.NewRequest("POST", u.String(), &b) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + precision := bp.Precision + if precision == "" { + precision = "ns" + } + + params := req.URL.Query() + params.Set("db", bp.Database) + params.Set("rp", bp.RetentionPolicy) + params.Set("precision", precision) + params.Set("consistency", bp.WriteConsistency) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + var err = fmt.Errorf(string(body)) + response.Err = err + return &response, err + } + + return nil, nil +} + +// WriteLineProtocol takes a string with line returns to delimit each write +// If successful, error is nil and Response is nil +// If an error occurs, Response may contain additional information if populated. +func (c *Client) WriteLineProtocol(data, database, retentionPolicy, precision, writeConsistency string) (*Response, error) { + u := c.url + u.Path = "write" + + r := strings.NewReader(data) + + req, err := http.NewRequest("POST", u.String(), r) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + params := req.URL.Query() + params.Set("db", database) + params.Set("rp", retentionPolicy) + params.Set("precision", precision) + params.Set("consistency", writeConsistency) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + err := fmt.Errorf(string(body)) + response.Err = err + return &response, err + } + + return nil, nil +} + +// Ping will check to see if the server is up +// Ping returns how long the request took, the version of the server it connected to, and an error if one occurred. +func (c *Client) Ping() (time.Duration, string, error) { + now := time.Now() + u := c.url + u.Path = "ping" + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return 0, "", err + } + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + + version := resp.Header.Get("X-Influxdb-Version") + return time.Since(now), version, nil +} + +// Structs + +// Message represents a user message. +type Message struct { + Level string `json:"level,omitempty"` + Text string `json:"text,omitempty"` +} + +// Result represents a resultset returned from a single statement. +type Result struct { + Series []models.Row + Messages []*Message + Err error +} + +// MarshalJSON encodes the result into JSON. +func (r *Result) MarshalJSON() ([]byte, error) { + // Define a struct that outputs "error" as a string. + var o struct { + Series []models.Row `json:"series,omitempty"` + Messages []*Message `json:"messages,omitempty"` + Err string `json:"error,omitempty"` + } + + // Copy fields to output struct. + o.Series = r.Series + o.Messages = r.Messages + if r.Err != nil { + o.Err = r.Err.Error() + } + + return json.Marshal(&o) +} + +// UnmarshalJSON decodes the data into the Result struct +func (r *Result) UnmarshalJSON(b []byte) error { + var o struct { + Series []models.Row `json:"series,omitempty"` + Messages []*Message `json:"messages,omitempty"` + Err string `json:"error,omitempty"` + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + err := dec.Decode(&o) + if err != nil { + return err + } + r.Series = o.Series + r.Messages = o.Messages + if o.Err != "" { + r.Err = errors.New(o.Err) + } + return nil +} + +// Response represents a list of statement results. +type Response struct { + Results []Result + Err error +} + +// MarshalJSON encodes the response into JSON. +func (r *Response) MarshalJSON() ([]byte, error) { + // Define a struct that outputs "error" as a string. + var o struct { + Results []Result `json:"results,omitempty"` + Err string `json:"error,omitempty"` + } + + // Copy fields to output struct. + o.Results = r.Results + if r.Err != nil { + o.Err = r.Err.Error() + } + + return json.Marshal(&o) +} + +// UnmarshalJSON decodes the data into the Response struct +func (r *Response) UnmarshalJSON(b []byte) error { + var o struct { + Results []Result `json:"results,omitempty"` + Err string `json:"error,omitempty"` + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + err := dec.Decode(&o) + if err != nil { + return err + } + r.Results = o.Results + if o.Err != "" { + r.Err = errors.New(o.Err) + } + return nil +} + +// Error returns the first error from any statement. +// Returns nil if no errors occurred on any statements. +func (r *Response) Error() error { + if r.Err != nil { + return r.Err + } + for _, result := range r.Results { + if result.Err != nil { + return result.Err + } + } + return nil +} + +// duplexReader reads responses and writes it to another writer while +// satisfying the reader interface. +type duplexReader struct { + r io.Reader + w io.Writer +} + +func (r *duplexReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if err == nil { + r.w.Write(p[:n]) + } + return n, err +} + +// ChunkedResponse represents a response from the server that +// uses chunking to stream the output. +type ChunkedResponse struct { + dec *json.Decoder + duplex *duplexReader + buf bytes.Buffer +} + +// NewChunkedResponse reads a stream and produces responses from the stream. +func NewChunkedResponse(r io.Reader) *ChunkedResponse { + resp := &ChunkedResponse{} + resp.duplex = &duplexReader{r: r, w: &resp.buf} + resp.dec = json.NewDecoder(resp.duplex) + resp.dec.UseNumber() + return resp +} + +// NextResponse reads the next line of the stream and returns a response. +func (r *ChunkedResponse) NextResponse() (*Response, error) { + var response Response + if err := r.dec.Decode(&response); err != nil { + if err == io.EOF { + return nil, nil + } + // A decoding error happened. This probably means the server crashed + // and sent a last-ditch error message to us. Ensure we have read the + // entirety of the connection to get any remaining error text. + io.Copy(ioutil.Discard, r.duplex) + return nil, errors.New(strings.TrimSpace(r.buf.String())) + } + r.buf.Reset() + return &response, nil +} + +// Point defines the fields that will be written to the database +// Measurement, Time, and Fields are required +// Precision can be specified if the time is in epoch format (integer). +// Valid values for Precision are n, u, ms, s, m, and h +type Point struct { + Measurement string + Tags map[string]string + Time time.Time + Fields map[string]interface{} + Precision string + Raw string +} + +// MarshalJSON will format the time in RFC3339Nano +// Precision is also ignored as it is only used for writing, not reading +// Or another way to say it is we always send back in nanosecond precision +func (p *Point) MarshalJSON() ([]byte, error) { + point := struct { + Measurement string `json:"measurement,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Time string `json:"time,omitempty"` + Fields map[string]interface{} `json:"fields,omitempty"` + Precision string `json:"precision,omitempty"` + }{ + Measurement: p.Measurement, + Tags: p.Tags, + Fields: p.Fields, + Precision: p.Precision, + } + // Let it omit empty if it's really zero + if !p.Time.IsZero() { + point.Time = p.Time.UTC().Format(time.RFC3339Nano) + } + return json.Marshal(&point) +} + +// MarshalString renders string representation of a Point with specified +// precision. The default precision is nanoseconds. +func (p *Point) MarshalString() string { + pt, err := models.NewPoint(p.Measurement, models.NewTags(p.Tags), p.Fields, p.Time) + if err != nil { + return "# ERROR: " + err.Error() + " " + p.Measurement + } + if p.Precision == "" || p.Precision == "ns" || p.Precision == "n" { + return pt.String() + } + return pt.PrecisionString(p.Precision) +} + +// UnmarshalJSON decodes the data into the Point struct +func (p *Point) UnmarshalJSON(b []byte) error { + var normal struct { + Measurement string `json:"measurement"` + Tags map[string]string `json:"tags"` + Time time.Time `json:"time"` + Precision string `json:"precision"` + Fields map[string]interface{} `json:"fields"` + } + var epoch struct { + Measurement string `json:"measurement"` + Tags map[string]string `json:"tags"` + Time *int64 `json:"time"` + Precision string `json:"precision"` + Fields map[string]interface{} `json:"fields"` + } + + if err := func() error { + var err error + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + if err = dec.Decode(&epoch); err != nil { + return err + } + // Convert from epoch to time.Time, but only if Time + // was actually set. + var ts time.Time + if epoch.Time != nil { + ts, err = EpochToTime(*epoch.Time, epoch.Precision) + if err != nil { + return err + } + } + p.Measurement = epoch.Measurement + p.Tags = epoch.Tags + p.Time = ts + p.Precision = epoch.Precision + p.Fields = normalizeFields(epoch.Fields) + return nil + }(); err == nil { + return nil + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + if err := dec.Decode(&normal); err != nil { + return err + } + normal.Time = SetPrecision(normal.Time, normal.Precision) + p.Measurement = normal.Measurement + p.Tags = normal.Tags + p.Time = normal.Time + p.Precision = normal.Precision + p.Fields = normalizeFields(normal.Fields) + + return nil +} + +// Remove any notion of json.Number +func normalizeFields(fields map[string]interface{}) map[string]interface{} { + newFields := map[string]interface{}{} + + for k, v := range fields { + switch v := v.(type) { + case json.Number: + jv, e := v.Float64() + if e != nil { + panic(fmt.Sprintf("unable to convert json.Number to float64: %s", e)) + } + newFields[k] = jv + default: + newFields[k] = v + } + } + return newFields +} + +// BatchPoints is used to send batched data in a single write. +// Database and Points are required +// If no retention policy is specified, it will use the databases default retention policy. +// If tags are specified, they will be "merged" with all points. If a point already has that tag, it will be ignored. +// If time is specified, it will be applied to any point with an empty time. +// Precision can be specified if the time is in epoch format (integer). +// Valid values for Precision are n, u, ms, s, m, and h +type BatchPoints struct { + Points []Point `json:"points,omitempty"` + Database string `json:"database,omitempty"` + RetentionPolicy string `json:"retentionPolicy,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Time time.Time `json:"time,omitempty"` + Precision string `json:"precision,omitempty"` + WriteConsistency string `json:"-"` +} + +// UnmarshalJSON decodes the data into the BatchPoints struct +func (bp *BatchPoints) UnmarshalJSON(b []byte) error { + var normal struct { + Points []Point `json:"points"` + Database string `json:"database"` + RetentionPolicy string `json:"retentionPolicy"` + Tags map[string]string `json:"tags"` + Time time.Time `json:"time"` + Precision string `json:"precision"` + } + var epoch struct { + Points []Point `json:"points"` + Database string `json:"database"` + RetentionPolicy string `json:"retentionPolicy"` + Tags map[string]string `json:"tags"` + Time *int64 `json:"time"` + Precision string `json:"precision"` + } + + if err := func() error { + var err error + if err = json.Unmarshal(b, &epoch); err != nil { + return err + } + // Convert from epoch to time.Time + var ts time.Time + if epoch.Time != nil { + ts, err = EpochToTime(*epoch.Time, epoch.Precision) + if err != nil { + return err + } + } + bp.Points = epoch.Points + bp.Database = epoch.Database + bp.RetentionPolicy = epoch.RetentionPolicy + bp.Tags = epoch.Tags + bp.Time = ts + bp.Precision = epoch.Precision + return nil + }(); err == nil { + return nil + } + + if err := json.Unmarshal(b, &normal); err != nil { + return err + } + normal.Time = SetPrecision(normal.Time, normal.Precision) + bp.Points = normal.Points + bp.Database = normal.Database + bp.RetentionPolicy = normal.RetentionPolicy + bp.Tags = normal.Tags + bp.Time = normal.Time + bp.Precision = normal.Precision + + return nil +} + +// utility functions + +// Addr provides the current url as a string of the server the client is connected to. +func (c *Client) Addr() string { + if c.unixSocket != "" { + return c.unixSocket + } + return c.url.String() +} + +// checkPointTypes ensures no unsupported types are submitted to influxdb, returning error if they are found. +func checkPointTypes(p Point) error { + for _, v := range p.Fields { + switch v.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool, string, nil: + return nil + default: + return fmt.Errorf("unsupported point type: %T", v) + } + } + return nil +} + +// helper functions + +// EpochToTime takes a unix epoch time and uses precision to return back a time.Time +func EpochToTime(epoch int64, precision string) (time.Time, error) { + if precision == "" { + precision = "s" + } + var t time.Time + switch precision { + case "h": + t = time.Unix(0, epoch*int64(time.Hour)) + case "m": + t = time.Unix(0, epoch*int64(time.Minute)) + case "s": + t = time.Unix(0, epoch*int64(time.Second)) + case "ms": + t = time.Unix(0, epoch*int64(time.Millisecond)) + case "u": + t = time.Unix(0, epoch*int64(time.Microsecond)) + case "n": + t = time.Unix(0, epoch) + default: + return time.Time{}, fmt.Errorf("Unknown precision %q", precision) + } + return t, nil +} + +// SetPrecision will round a time to the specified precision +func SetPrecision(t time.Time, precision string) time.Time { + switch precision { + case "n": + case "u": + return t.Round(time.Microsecond) + case "ms": + return t.Round(time.Millisecond) + case "s": + return t.Round(time.Second) + case "m": + return t.Round(time.Minute) + case "h": + return t.Round(time.Hour) + } + return t +} diff --git a/vendor/github.com/influxdata/influxdb/client/v2/client.go b/vendor/github.com/influxdata/influxdb/client/v2/client.go new file mode 100644 index 0000000000..77d44f2b34 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/v2/client.go @@ -0,0 +1,635 @@ +// Package client (v2) is the current official Go client for InfluxDB. +package client // import "github.com/influxdata/influxdb/client/v2" + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/models" +) + +// HTTPConfig is the config data needed to create an HTTP Client. +type HTTPConfig struct { + // Addr should be of the form "http://host:port" + // or "http://[ipv6-host%zone]:port". + Addr string + + // Username is the influxdb username, optional. + Username string + + // Password is the influxdb password, optional. + Password string + + // UserAgent is the http User Agent, defaults to "InfluxDBClient". + UserAgent string + + // Timeout for influxdb writes, defaults to no timeout. + Timeout time.Duration + + // InsecureSkipVerify gets passed to the http client, if true, it will + // skip https certificate verification. Defaults to false. + InsecureSkipVerify bool + + // TLSConfig allows the user to set their own TLS config for the HTTP + // Client. If set, this option overrides InsecureSkipVerify. + TLSConfig *tls.Config +} + +// BatchPointsConfig is the config data needed to create an instance of the BatchPoints struct. +type BatchPointsConfig struct { + // Precision is the write precision of the points, defaults to "ns". + Precision string + + // Database is the database to write points to. + Database string + + // RetentionPolicy is the retention policy of the points. + RetentionPolicy string + + // Write consistency is the number of servers required to confirm write. + WriteConsistency string +} + +// Client is a client interface for writing & querying the database. +type Client interface { + // Ping checks that status of cluster, and will always return 0 time and no + // error for UDP clients. + Ping(timeout time.Duration) (time.Duration, string, error) + + // Write takes a BatchPoints object and writes all Points to InfluxDB. + Write(bp BatchPoints) error + + // Query makes an InfluxDB Query on the database. This will fail if using + // the UDP client. + Query(q Query) (*Response, error) + + // Close releases any resources a Client may be using. + Close() error +} + +// NewHTTPClient returns a new Client from the provided config. +// Client is safe for concurrent use by multiple goroutines. +func NewHTTPClient(conf HTTPConfig) (Client, error) { + if conf.UserAgent == "" { + conf.UserAgent = "InfluxDBClient" + } + + u, err := url.Parse(conf.Addr) + if err != nil { + return nil, err + } else if u.Scheme != "http" && u.Scheme != "https" { + m := fmt.Sprintf("Unsupported protocol scheme: %s, your address"+ + " must start with http:// or https://", u.Scheme) + return nil, errors.New(m) + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: conf.InsecureSkipVerify, + }, + } + if conf.TLSConfig != nil { + tr.TLSClientConfig = conf.TLSConfig + } + return &client{ + url: *u, + username: conf.Username, + password: conf.Password, + useragent: conf.UserAgent, + httpClient: &http.Client{ + Timeout: conf.Timeout, + Transport: tr, + }, + transport: tr, + }, nil +} + +// Ping will check to see if the server is up with an optional timeout on waiting for leader. +// Ping returns how long the request took, the version of the server it connected to, and an error if one occurred. +func (c *client) Ping(timeout time.Duration) (time.Duration, string, error) { + now := time.Now() + u := c.url + u.Path = "ping" + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return 0, "", err + } + + req.Header.Set("User-Agent", c.useragent) + + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + if timeout > 0 { + params := req.URL.Query() + params.Set("wait_for_leader", fmt.Sprintf("%.0fs", timeout.Seconds())) + req.URL.RawQuery = params.Encode() + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, "", err + } + + if resp.StatusCode != http.StatusNoContent { + var err = fmt.Errorf(string(body)) + return 0, "", err + } + + version := resp.Header.Get("X-Influxdb-Version") + return time.Since(now), version, nil +} + +// Close releases the client's resources. +func (c *client) Close() error { + c.transport.CloseIdleConnections() + return nil +} + +// client is safe for concurrent use as the fields are all read-only +// once the client is instantiated. +type client struct { + // N.B - if url.UserInfo is accessed in future modifications to the + // methods on client, you will need to syncronise access to url. + url url.URL + username string + password string + useragent string + httpClient *http.Client + transport *http.Transport +} + +// BatchPoints is an interface into a batched grouping of points to write into +// InfluxDB together. BatchPoints is NOT thread-safe, you must create a separate +// batch for each goroutine. +type BatchPoints interface { + // AddPoint adds the given point to the Batch of points. + AddPoint(p *Point) + // AddPoints adds the given points to the Batch of points. + AddPoints(ps []*Point) + // Points lists the points in the Batch. + Points() []*Point + + // Precision returns the currently set precision of this Batch. + Precision() string + // SetPrecision sets the precision of this batch. + SetPrecision(s string) error + + // Database returns the currently set database of this Batch. + Database() string + // SetDatabase sets the database of this Batch. + SetDatabase(s string) + + // WriteConsistency returns the currently set write consistency of this Batch. + WriteConsistency() string + // SetWriteConsistency sets the write consistency of this Batch. + SetWriteConsistency(s string) + + // RetentionPolicy returns the currently set retention policy of this Batch. + RetentionPolicy() string + // SetRetentionPolicy sets the retention policy of this Batch. + SetRetentionPolicy(s string) +} + +// NewBatchPoints returns a BatchPoints interface based on the given config. +func NewBatchPoints(conf BatchPointsConfig) (BatchPoints, error) { + if conf.Precision == "" { + conf.Precision = "ns" + } + if _, err := time.ParseDuration("1" + conf.Precision); err != nil { + return nil, err + } + bp := &batchpoints{ + database: conf.Database, + precision: conf.Precision, + retentionPolicy: conf.RetentionPolicy, + writeConsistency: conf.WriteConsistency, + } + return bp, nil +} + +type batchpoints struct { + points []*Point + database string + precision string + retentionPolicy string + writeConsistency string +} + +func (bp *batchpoints) AddPoint(p *Point) { + bp.points = append(bp.points, p) +} + +func (bp *batchpoints) AddPoints(ps []*Point) { + bp.points = append(bp.points, ps...) +} + +func (bp *batchpoints) Points() []*Point { + return bp.points +} + +func (bp *batchpoints) Precision() string { + return bp.precision +} + +func (bp *batchpoints) Database() string { + return bp.database +} + +func (bp *batchpoints) WriteConsistency() string { + return bp.writeConsistency +} + +func (bp *batchpoints) RetentionPolicy() string { + return bp.retentionPolicy +} + +func (bp *batchpoints) SetPrecision(p string) error { + if _, err := time.ParseDuration("1" + p); err != nil { + return err + } + bp.precision = p + return nil +} + +func (bp *batchpoints) SetDatabase(db string) { + bp.database = db +} + +func (bp *batchpoints) SetWriteConsistency(wc string) { + bp.writeConsistency = wc +} + +func (bp *batchpoints) SetRetentionPolicy(rp string) { + bp.retentionPolicy = rp +} + +// Point represents a single data point. +type Point struct { + pt models.Point +} + +// NewPoint returns a point with the given timestamp. If a timestamp is not +// given, then data is sent to the database without a timestamp, in which case +// the server will assign local time upon reception. NOTE: it is recommended to +// send data with a timestamp. +func NewPoint( + name string, + tags map[string]string, + fields map[string]interface{}, + t ...time.Time, +) (*Point, error) { + var T time.Time + if len(t) > 0 { + T = t[0] + } + + pt, err := models.NewPoint(name, models.NewTags(tags), fields, T) + if err != nil { + return nil, err + } + return &Point{ + pt: pt, + }, nil +} + +// String returns a line-protocol string of the Point. +func (p *Point) String() string { + return p.pt.String() +} + +// PrecisionString returns a line-protocol string of the Point, +// with the timestamp formatted for the given precision. +func (p *Point) PrecisionString(precison string) string { + return p.pt.PrecisionString(precison) +} + +// Name returns the measurement name of the point. +func (p *Point) Name() string { + return string(p.pt.Name()) +} + +// Tags returns the tags associated with the point. +func (p *Point) Tags() map[string]string { + return p.pt.Tags().Map() +} + +// Time return the timestamp for the point. +func (p *Point) Time() time.Time { + return p.pt.Time() +} + +// UnixNano returns timestamp of the point in nanoseconds since Unix epoch. +func (p *Point) UnixNano() int64 { + return p.pt.UnixNano() +} + +// Fields returns the fields for the point. +func (p *Point) Fields() (map[string]interface{}, error) { + return p.pt.Fields() +} + +// NewPointFrom returns a point from the provided models.Point. +func NewPointFrom(pt models.Point) *Point { + return &Point{pt: pt} +} + +func (c *client) Write(bp BatchPoints) error { + var b bytes.Buffer + + for _, p := range bp.Points() { + if _, err := b.WriteString(p.pt.PrecisionString(bp.Precision())); err != nil { + return err + } + + if err := b.WriteByte('\n'); err != nil { + return err + } + } + + u := c.url + u.Path = "write" + req, err := http.NewRequest("POST", u.String(), &b) + if err != nil { + return err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.useragent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + params := req.URL.Query() + params.Set("db", bp.Database()) + params.Set("rp", bp.RetentionPolicy()) + params.Set("precision", bp.Precision()) + params.Set("consistency", bp.WriteConsistency()) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + var err = fmt.Errorf(string(body)) + return err + } + + return nil +} + +// Query defines a query to send to the server. +type Query struct { + Command string + Database string + Precision string + Chunked bool + ChunkSize int + Parameters map[string]interface{} +} + +// NewQuery returns a query object. +// The database and precision arguments can be empty strings if they are not needed for the query. +func NewQuery(command, database, precision string) Query { + return Query{ + Command: command, + Database: database, + Precision: precision, + Parameters: make(map[string]interface{}), + } +} + +// NewQueryWithParameters returns a query object. +// The database and precision arguments can be empty strings if they are not needed for the query. +// parameters is a map of the parameter names used in the command to their values. +func NewQueryWithParameters(command, database, precision string, parameters map[string]interface{}) Query { + return Query{ + Command: command, + Database: database, + Precision: precision, + Parameters: parameters, + } +} + +// Response represents a list of statement results. +type Response struct { + Results []Result + Err string `json:"error,omitempty"` +} + +// Error returns the first error from any statement. +// It returns nil if no errors occurred on any statements. +func (r *Response) Error() error { + if r.Err != "" { + return fmt.Errorf(r.Err) + } + for _, result := range r.Results { + if result.Err != "" { + return fmt.Errorf(result.Err) + } + } + return nil +} + +// Message represents a user message. +type Message struct { + Level string + Text string +} + +// Result represents a resultset returned from a single statement. +type Result struct { + Series []models.Row + Messages []*Message + Err string `json:"error,omitempty"` +} + +// Query sends a command to the server and returns the Response. +func (c *client) Query(q Query) (*Response, error) { + u := c.url + u.Path = "query" + + jsonParameters, err := json.Marshal(q.Parameters) + + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.useragent) + + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + params := req.URL.Query() + params.Set("q", q.Command) + params.Set("db", q.Database) + params.Set("params", string(jsonParameters)) + if q.Chunked { + params.Set("chunked", "true") + if q.ChunkSize > 0 { + params.Set("chunk_size", strconv.Itoa(q.ChunkSize)) + } + } + + if q.Precision != "" { + params.Set("epoch", q.Precision) + } + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // If we lack a X-Influxdb-Version header, then we didn't get a response from influxdb + // but instead some other service. If the error code is also a 500+ code, then some + // downstream loadbalancer/proxy/etc had an issue and we should report that. + if resp.Header.Get("X-Influxdb-Version") == "" && resp.StatusCode >= http.StatusInternalServerError { + body, err := ioutil.ReadAll(resp.Body) + if err != nil || len(body) == 0 { + return nil, fmt.Errorf("received status code %d from downstream server", resp.StatusCode) + } + + return nil, fmt.Errorf("received status code %d from downstream server, with response body: %q", resp.StatusCode, body) + } + + // If we get an unexpected content type, then it is also not from influx direct and therefore + // we want to know what we received and what status code was returned for debugging purposes. + if cType, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")); cType != "application/json" { + // Read up to 1kb of the body to help identify downstream errors and limit the impact of things + // like downstream serving a large file + body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024)) + if err != nil || len(body) == 0 { + return nil, fmt.Errorf("expected json response, got empty body, with status: %v", resp.StatusCode) + } + + return nil, fmt.Errorf("expected json response, got %q, with status: %v and response body: %q", cType, resp.StatusCode, body) + } + + var response Response + if q.Chunked { + cr := NewChunkedResponse(resp.Body) + for { + r, err := cr.NextResponse() + if err != nil { + // If we got an error while decoding the response, send that back. + return nil, err + } + + if r == nil { + break + } + + response.Results = append(response.Results, r.Results...) + if r.Err != "" { + response.Err = r.Err + break + } + } + } else { + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + decErr := dec.Decode(&response) + + // ignore this error if we got an invalid status code + if decErr != nil && decErr.Error() == "EOF" && resp.StatusCode != http.StatusOK { + decErr = nil + } + // If we got a valid decode error, send that back + if decErr != nil { + return nil, fmt.Errorf("unable to decode json: received status code %d err: %s", resp.StatusCode, decErr) + } + } + + // If we don't have an error in our json response, and didn't get statusOK + // then send back an error + if resp.StatusCode != http.StatusOK && response.Error() == nil { + return &response, fmt.Errorf("received status code %d from server", resp.StatusCode) + } + return &response, nil +} + +// duplexReader reads responses and writes it to another writer while +// satisfying the reader interface. +type duplexReader struct { + r io.Reader + w io.Writer +} + +func (r *duplexReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if err == nil { + r.w.Write(p[:n]) + } + return n, err +} + +// ChunkedResponse represents a response from the server that +// uses chunking to stream the output. +type ChunkedResponse struct { + dec *json.Decoder + duplex *duplexReader + buf bytes.Buffer +} + +// NewChunkedResponse reads a stream and produces responses from the stream. +func NewChunkedResponse(r io.Reader) *ChunkedResponse { + resp := &ChunkedResponse{} + resp.duplex = &duplexReader{r: r, w: &resp.buf} + resp.dec = json.NewDecoder(resp.duplex) + resp.dec.UseNumber() + return resp +} + +// NextResponse reads the next line of the stream and returns a response. +func (r *ChunkedResponse) NextResponse() (*Response, error) { + var response Response + + if err := r.dec.Decode(&response); err != nil { + if err == io.EOF { + return nil, nil + } + // A decoding error happened. This probably means the server crashed + // and sent a last-ditch error message to us. Ensure we have read the + // entirety of the connection to get any remaining error text. + io.Copy(ioutil.Discard, r.duplex) + return nil, errors.New(strings.TrimSpace(r.buf.String())) + } + + r.buf.Reset() + return &response, nil +} diff --git a/vendor/github.com/influxdata/influxdb/client/v2/udp.go b/vendor/github.com/influxdata/influxdb/client/v2/udp.go new file mode 100644 index 0000000000..779a28b33f --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/v2/udp.go @@ -0,0 +1,112 @@ +package client + +import ( + "fmt" + "io" + "net" + "time" +) + +const ( + // UDPPayloadSize is a reasonable default payload size for UDP packets that + // could be travelling over the internet. + UDPPayloadSize = 512 +) + +// UDPConfig is the config data needed to create a UDP Client. +type UDPConfig struct { + // Addr should be of the form "host:port" + // or "[ipv6-host%zone]:port". + Addr string + + // PayloadSize is the maximum size of a UDP client message, optional + // Tune this based on your network. Defaults to UDPPayloadSize. + PayloadSize int +} + +// NewUDPClient returns a client interface for writing to an InfluxDB UDP +// service from the given config. +func NewUDPClient(conf UDPConfig) (Client, error) { + var udpAddr *net.UDPAddr + udpAddr, err := net.ResolveUDPAddr("udp", conf.Addr) + if err != nil { + return nil, err + } + + conn, err := net.DialUDP("udp", nil, udpAddr) + if err != nil { + return nil, err + } + + payloadSize := conf.PayloadSize + if payloadSize == 0 { + payloadSize = UDPPayloadSize + } + + return &udpclient{ + conn: conn, + payloadSize: payloadSize, + }, nil +} + +// Close releases the udpclient's resources. +func (uc *udpclient) Close() error { + return uc.conn.Close() +} + +type udpclient struct { + conn io.WriteCloser + payloadSize int +} + +func (uc *udpclient) Write(bp BatchPoints) error { + var b = make([]byte, 0, uc.payloadSize) // initial buffer size, it will grow as needed + var d, _ = time.ParseDuration("1" + bp.Precision()) + + var delayedError error + + var checkBuffer = func(n int) { + if len(b) > 0 && len(b)+n > uc.payloadSize { + if _, err := uc.conn.Write(b); err != nil { + delayedError = err + } + b = b[:0] + } + } + + for _, p := range bp.Points() { + p.pt.Round(d) + pointSize := p.pt.StringSize() + 1 // include newline in size + //point := p.pt.RoundedString(d) + "\n" + + checkBuffer(pointSize) + + if p.Time().IsZero() || pointSize <= uc.payloadSize { + b = p.pt.AppendString(b) + b = append(b, '\n') + continue + } + + points := p.pt.Split(uc.payloadSize - 1) // account for newline character + for _, sp := range points { + checkBuffer(sp.StringSize() + 1) + b = sp.AppendString(b) + b = append(b, '\n') + } + } + + if len(b) > 0 { + if _, err := uc.conn.Write(b); err != nil { + return err + } + } + return delayedError +} + +func (uc *udpclient) Query(q Query) (*Response, error) { + return nil, fmt.Errorf("Querying via UDP is not supported") +} + +func (uc *udpclient) Ping(timeout time.Duration) (time.Duration, string, error) { + return 0, "", nil +} diff --git a/vendor/github.com/influxdata/influxdb/models/consistency.go b/vendor/github.com/influxdata/influxdb/models/consistency.go new file mode 100644 index 0000000000..2a3269bca1 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/consistency.go @@ -0,0 +1,48 @@ +package models + +import ( + "errors" + "strings" +) + +// ConsistencyLevel represent a required replication criteria before a write can +// be returned as successful. +// +// The consistency level is handled in open-source InfluxDB but only applicable to clusters. +type ConsistencyLevel int + +const ( + // ConsistencyLevelAny allows for hinted handoff, potentially no write happened yet. + ConsistencyLevelAny ConsistencyLevel = iota + + // ConsistencyLevelOne requires at least one data node acknowledged a write. + ConsistencyLevelOne + + // ConsistencyLevelQuorum requires a quorum of data nodes to acknowledge a write. + ConsistencyLevelQuorum + + // ConsistencyLevelAll requires all data nodes to acknowledge a write. + ConsistencyLevelAll +) + +var ( + // ErrInvalidConsistencyLevel is returned when parsing the string version + // of a consistency level. + ErrInvalidConsistencyLevel = errors.New("invalid consistency level") +) + +// ParseConsistencyLevel converts a consistency level string to the corresponding ConsistencyLevel const. +func ParseConsistencyLevel(level string) (ConsistencyLevel, error) { + switch strings.ToLower(level) { + case "any": + return ConsistencyLevelAny, nil + case "one": + return ConsistencyLevelOne, nil + case "quorum": + return ConsistencyLevelQuorum, nil + case "all": + return ConsistencyLevelAll, nil + default: + return 0, ErrInvalidConsistencyLevel + } +} diff --git a/vendor/github.com/influxdata/influxdb/models/inline_fnv.go b/vendor/github.com/influxdata/influxdb/models/inline_fnv.go new file mode 100644 index 0000000000..eec1ae8b01 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/inline_fnv.go @@ -0,0 +1,32 @@ +package models // import "github.com/influxdata/influxdb/models" + +// from stdlib hash/fnv/fnv.go +const ( + prime64 = 1099511628211 + offset64 = 14695981039346656037 +) + +// InlineFNV64a is an alloc-free port of the standard library's fnv64a. +// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function. +type InlineFNV64a uint64 + +// NewInlineFNV64a returns a new instance of InlineFNV64a. +func NewInlineFNV64a() InlineFNV64a { + return offset64 +} + +// Write adds data to the running hash. +func (s *InlineFNV64a) Write(data []byte) (int, error) { + hash := uint64(*s) + for _, c := range data { + hash ^= uint64(c) + hash *= prime64 + } + *s = InlineFNV64a(hash) + return len(data), nil +} + +// Sum64 returns the uint64 of the current resulting hash. +func (s *InlineFNV64a) Sum64() uint64 { + return uint64(*s) +} diff --git a/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go b/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go new file mode 100644 index 0000000000..8db4837384 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go @@ -0,0 +1,44 @@ +package models // import "github.com/influxdata/influxdb/models" + +import ( + "reflect" + "strconv" + "unsafe" +) + +// parseIntBytes is a zero-alloc wrapper around strconv.ParseInt. +func parseIntBytes(b []byte, base int, bitSize int) (i int64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseInt(s, base, bitSize) +} + +// parseUintBytes is a zero-alloc wrapper around strconv.ParseUint. +func parseUintBytes(b []byte, base int, bitSize int) (i uint64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseUint(s, base, bitSize) +} + +// parseFloatBytes is a zero-alloc wrapper around strconv.ParseFloat. +func parseFloatBytes(b []byte, bitSize int) (float64, error) { + s := unsafeBytesToString(b) + return strconv.ParseFloat(s, bitSize) +} + +// parseBoolBytes is a zero-alloc wrapper around strconv.ParseBool. +func parseBoolBytes(b []byte) (bool, error) { + return strconv.ParseBool(unsafeBytesToString(b)) +} + +// unsafeBytesToString converts a []byte to a string without a heap allocation. +// +// It is unsafe, and is intended to prepare input to short-lived functions +// that require strings. +func unsafeBytesToString(in []byte) string { + src := *(*reflect.SliceHeader)(unsafe.Pointer(&in)) + dst := reflect.StringHeader{ + Data: src.Data, + Len: src.Len, + } + s := *(*string)(unsafe.Pointer(&dst)) + return s +} diff --git a/vendor/github.com/influxdata/influxdb/models/points.go b/vendor/github.com/influxdata/influxdb/models/points.go new file mode 100644 index 0000000000..ad80a816bf --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/points.go @@ -0,0 +1,2337 @@ +// Package models implements basic objects used throughout the TICK stack. +package models // import "github.com/influxdata/influxdb/models" + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/pkg/escape" +) + +var ( + measurementEscapeCodes = map[byte][]byte{ + ',': []byte(`\,`), + ' ': []byte(`\ `), + } + + tagEscapeCodes = map[byte][]byte{ + ',': []byte(`\,`), + ' ': []byte(`\ `), + '=': []byte(`\=`), + } + + // ErrPointMustHaveAField is returned when operating on a point that does not have any fields. + ErrPointMustHaveAField = errors.New("point without fields is unsupported") + + // ErrInvalidNumber is returned when a number is expected but not provided. + ErrInvalidNumber = errors.New("invalid number") + + // ErrInvalidPoint is returned when a point cannot be parsed correctly. + ErrInvalidPoint = errors.New("point is invalid") +) + +const ( + // MaxKeyLength is the largest allowed size of the combined measurement and tag keys. + MaxKeyLength = 65535 +) + +// enableUint64Support will enable uint64 support if set to true. +var enableUint64Support = false + +// EnableUintSupport manually enables uint support for the point parser. +// This function will be removed in the future and only exists for unit tests during the +// transition. +func EnableUintSupport() { + enableUint64Support = true +} + +// Point defines the values that will be written to the database. +type Point interface { + // Name return the measurement name for the point. + Name() []byte + + // SetName updates the measurement name for the point. + SetName(string) + + // Tags returns the tag set for the point. + Tags() Tags + + // AddTag adds or replaces a tag value for a point. + AddTag(key, value string) + + // SetTags replaces the tags for the point. + SetTags(tags Tags) + + // HasTag returns true if the tag exists for the point. + HasTag(tag []byte) bool + + // Fields returns the fields for the point. + Fields() (Fields, error) + + // Time return the timestamp for the point. + Time() time.Time + + // SetTime updates the timestamp for the point. + SetTime(t time.Time) + + // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. + UnixNano() int64 + + // HashID returns a non-cryptographic checksum of the point's key. + HashID() uint64 + + // Key returns the key (measurement joined with tags) of the point. + Key() []byte + + // String returns a string representation of the point. If there is a + // timestamp associated with the point then it will be specified with the default + // precision of nanoseconds. + String() string + + // MarshalBinary returns a binary representation of the point. + MarshalBinary() ([]byte, error) + + // PrecisionString returns a string representation of the point. If there + // is a timestamp associated with the point then it will be specified in the + // given unit. + PrecisionString(precision string) string + + // RoundedString returns a string representation of the point. If there + // is a timestamp associated with the point, then it will be rounded to the + // given duration. + RoundedString(d time.Duration) string + + // Split will attempt to return multiple points with the same timestamp whose + // string representations are no longer than size. Points with a single field or + // a point without a timestamp may exceed the requested size. + Split(size int) []Point + + // Round will round the timestamp of the point to the given duration. + Round(d time.Duration) + + // StringSize returns the length of the string that would be returned by String(). + StringSize() int + + // AppendString appends the result of String() to the provided buffer and returns + // the result, potentially reducing string allocations. + AppendString(buf []byte) []byte + + // FieldIterator retuns a FieldIterator that can be used to traverse the + // fields of a point without constructing the in-memory map. + FieldIterator() FieldIterator +} + +// FieldType represents the type of a field. +type FieldType int + +const ( + // Integer indicates the field's type is integer. + Integer FieldType = iota + + // Float indicates the field's type is float. + Float + + // Boolean indicates the field's type is boolean. + Boolean + + // String indicates the field's type is string. + String + + // Empty is used to indicate that there is no field. + Empty + + // Unsigned indicates the field's type is an unsigned integer. + Unsigned +) + +// FieldIterator provides a low-allocation interface to iterate through a point's fields. +type FieldIterator interface { + // Next indicates whether there any fields remaining. + Next() bool + + // FieldKey returns the key of the current field. + FieldKey() []byte + + // Type returns the FieldType of the current field. + Type() FieldType + + // StringValue returns the string value of the current field. + StringValue() string + + // IntegerValue returns the integer value of the current field. + IntegerValue() (int64, error) + + // UnsignedValue returns the unsigned value of the current field. + UnsignedValue() (uint64, error) + + // BooleanValue returns the boolean value of the current field. + BooleanValue() (bool, error) + + // FloatValue returns the float value of the current field. + FloatValue() (float64, error) + + // Reset resets the iterator to its initial state. + Reset() +} + +// Points represents a sortable list of points by timestamp. +type Points []Point + +// Len implements sort.Interface. +func (a Points) Len() int { return len(a) } + +// Less implements sort.Interface. +func (a Points) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } + +// Swap implements sort.Interface. +func (a Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// point is the default implementation of Point. +type point struct { + time time.Time + + // text encoding of measurement and tags + // key must always be stored sorted by tags, if the original line was not sorted, + // we need to resort it + key []byte + + // text encoding of field data + fields []byte + + // text encoding of timestamp + ts []byte + + // cached version of parsed fields from data + cachedFields map[string]interface{} + + // cached version of parsed name from key + cachedName string + + // cached version of parsed tags + cachedTags Tags + + it fieldIterator +} + +// type assertions +var ( + _ Point = (*point)(nil) + _ FieldIterator = (*point)(nil) +) + +const ( + // the number of characters for the largest possible int64 (9223372036854775807) + maxInt64Digits = 19 + + // the number of characters for the smallest possible int64 (-9223372036854775808) + minInt64Digits = 20 + + // the number of characters for the largest possible uint64 (18446744073709551615) + maxUint64Digits = 20 + + // the number of characters required for the largest float64 before a range check + // would occur during parsing + maxFloat64Digits = 25 + + // the number of characters required for smallest float64 before a range check occur + // would occur during parsing + minFloat64Digits = 27 +) + +// ParsePoints returns a slice of Points from a text representation of a point +// with each point separated by newlines. If any points fail to parse, a non-nil error +// will be returned in addition to the points that parsed successfully. +func ParsePoints(buf []byte) ([]Point, error) { + return ParsePointsWithPrecision(buf, time.Now().UTC(), "n") +} + +// ParsePointsString is identical to ParsePoints but accepts a string. +func ParsePointsString(buf string) ([]Point, error) { + return ParsePoints([]byte(buf)) +} + +// ParseKey returns the measurement name and tags from a point. +// +// NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParseKey(buf []byte) (string, Tags) { + meas, tags := ParseKeyBytes(buf) + return string(meas), tags +} + +func ParseKeyBytes(buf []byte) ([]byte, Tags) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + + var tags Tags + if state == tagKeyState { + tags = parseTags(buf) + // scanMeasurement returns the location of the comma if there are tags, strip that off + return buf[:i-1], tags + } + return buf[:i], tags +} + +func ParseTags(buf []byte) Tags { + return parseTags(buf) +} + +func ParseName(buf []byte) ([]byte, error) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + if state == tagKeyState { + return buf[:i-1], nil + } + return buf[:i], nil +} + +// ParsePointsWithPrecision is similar to ParsePoints, but allows the +// caller to provide a precision for time. +// +// NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { + points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) + var ( + pos int + block []byte + failed []string + ) + for pos < len(buf) { + pos, block = scanLine(buf, pos) + pos++ + + if len(block) == 0 { + continue + } + + // lines which start with '#' are comments + start := skipWhitespace(block, 0) + + // If line is all whitespace, just skip it + if start >= len(block) { + continue + } + + if block[start] == '#' { + continue + } + + // strip the newline if one is present + if block[len(block)-1] == '\n' { + block = block[:len(block)-1] + } + + pt, err := parsePoint(block[start:], defaultTime, precision) + if err != nil { + failed = append(failed, fmt.Sprintf("unable to parse '%s': %v", string(block[start:]), err)) + } else { + points = append(points, pt) + } + + } + if len(failed) > 0 { + return points, fmt.Errorf("%s", strings.Join(failed, "\n")) + } + return points, nil + +} + +func parsePoint(buf []byte, defaultTime time.Time, precision string) (Point, error) { + // scan the first block which is measurement[,tag1=value1,tag2=value=2...] + pos, key, err := scanKey(buf, 0) + if err != nil { + return nil, err + } + + // measurement name is required + if len(key) == 0 { + return nil, fmt.Errorf("missing measurement") + } + + if len(key) > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) + } + + // scan the second block is which is field1=value1[,field2=value2,...] + pos, fields, err := scanFields(buf, pos) + if err != nil { + return nil, err + } + + // at least one field is required + if len(fields) == 0 { + return nil, fmt.Errorf("missing fields") + } + + var maxKeyErr error + walkFields(fields, func(k, v []byte) bool { + if sz := seriesKeySize(key, k); sz > MaxKeyLength { + maxKeyErr = fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + return false + } + return true + }) + + if maxKeyErr != nil { + return nil, maxKeyErr + } + + // scan the last block which is an optional integer timestamp + pos, ts, err := scanTime(buf, pos) + if err != nil { + return nil, err + } + + pt := &point{ + key: key, + fields: fields, + ts: ts, + } + + if len(ts) == 0 { + pt.time = defaultTime + pt.SetPrecision(precision) + } else { + ts, err := parseIntBytes(ts, 10, 64) + if err != nil { + return nil, err + } + pt.time, err = SafeCalcTime(ts, precision) + if err != nil { + return nil, err + } + + // Determine if there are illegal non-whitespace characters after the + // timestamp block. + for pos < len(buf) { + if buf[pos] != ' ' { + return nil, ErrInvalidPoint + } + pos++ + } + } + return pt, nil +} + +// GetPrecisionMultiplier will return a multiplier for the precision specified. +func GetPrecisionMultiplier(precision string) int64 { + d := time.Nanosecond + switch precision { + case "u": + d = time.Microsecond + case "ms": + d = time.Millisecond + case "s": + d = time.Second + case "m": + d = time.Minute + case "h": + d = time.Hour + } + return int64(d) +} + +// scanKey scans buf starting at i for the measurement and tag portion of the point. +// It returns the ending position and the byte slice of key within buf. If there +// are tags, they will be sorted if they are not already. +func scanKey(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + + i = start + + // Determines whether the tags are sort, assume they are + sorted := true + + // indices holds the indexes within buf of the start of each tag. For example, + // a buf of 'cpu,host=a,region=b,zone=c' would have indices slice of [4,11,20] + // which indicates that the first tag starts at buf[4], seconds at buf[11], and + // last at buf[20] + indices := make([]int, 100) + + // tracks how many commas we've seen so we know how many values are indices. + // Since indices is an arbitrarily large slice, + // we need to know how many values in the buffer are in use. + commas := 0 + + // First scan the Point's measurement. + state, i, err := scanMeasurement(buf, i) + if err != nil { + return i, buf[start:i], err + } + + // Optionally scan tags if needed. + if state == tagKeyState { + i, commas, indices, err = scanTags(buf, i, indices) + if err != nil { + return i, buf[start:i], err + } + } + + // Now we know where the key region is within buf, and the location of tags, we + // need to determine if duplicate tags exist and if the tags are sorted. This iterates + // over the list comparing each tag in the sequence with each other. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:indices[j+1]-1], 0, '=') + _, right := scanTo(buf[indices[j+1]:indices[j+2]-1], 0, '=') + + // If left is greater than right, the tags are not sorted. We do not have to + // continue because the short path no longer works. + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if cmp := bytes.Compare(left, right); cmp > 0 { + sorted = false + break + } else if cmp == 0 { + return i, buf[start:i], fmt.Errorf("duplicate tags") + } + } + + // If the tags are not sorted, then sort them. This sort is inline and + // uses the tag indices we created earlier. The actual buffer is not sorted, the + // indices are using the buffer for value comparison. After the indices are sorted, + // the buffer is reconstructed from the sorted indices. + if !sorted && commas > 0 { + // Get the measurement name for later + measurement := buf[start : indices[0]-1] + + // Sort the indices + indices := indices[:commas] + insertionSort(0, commas, buf, indices) + + // Create a new key using the measurement and sorted indices + b := make([]byte, len(buf[start:i])) + pos := copy(b, measurement) + for _, i := range indices { + b[pos] = ',' + pos++ + _, v := scanToSpaceOr(buf, i, ',') + pos += copy(b[pos:], v) + } + + // Check again for duplicate tags now that the tags are sorted. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:], 0, '=') + _, right := scanTo(buf[indices[j+1]:], 0, '=') + + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if bytes.Equal(left, right) { + return i, b, fmt.Errorf("duplicate tags") + } + } + + return i, b, nil + } + + return i, buf[start:i], nil +} + +// The following constants allow us to specify which state to move to +// next, when scanning sections of a Point. +const ( + tagKeyState = iota + tagValueState + fieldsState +) + +// scanMeasurement examines the measurement part of a Point, returning +// the next state to move to, and the current location in the buffer. +func scanMeasurement(buf []byte, i int) (int, int, error) { + // Check first byte of measurement, anything except a comma is fine. + // It can't be a space, since whitespace is stripped prior to this + // function call. + if i >= len(buf) || buf[i] == ',' { + return -1, i, fmt.Errorf("missing measurement") + } + + for { + i++ + if i >= len(buf) { + // cpu + return -1, i, fmt.Errorf("missing fields") + } + + if buf[i-1] == '\\' { + // Skip character (it's escaped). + continue + } + + // Unescaped comma; move onto scanning the tags. + if buf[i] == ',' { + return tagKeyState, i + 1, nil + } + + // Unescaped space; move onto scanning the fields. + if buf[i] == ' ' { + // cpu value=1.0 + return fieldsState, i, nil + } + } +} + +// scanTags examines all the tags in a Point, keeping track of and +// returning the updated indices slice, number of commas and location +// in buf where to start examining the Point fields. +func scanTags(buf []byte, i int, indices []int) (int, int, []int, error) { + var ( + err error + commas int + state = tagKeyState + ) + + for { + switch state { + case tagKeyState: + // Grow our indices slice if we have too many tags. + if commas >= len(indices) { + newIndics := make([]int, cap(indices)*2) + copy(newIndics, indices) + indices = newIndics + } + indices[commas] = i + commas++ + + i, err = scanTagsKey(buf, i) + state = tagValueState // tag value always follows a tag key + case tagValueState: + state, i, err = scanTagsValue(buf, i) + case fieldsState: + indices[commas] = i + 1 + return i, commas, indices, nil + } + + if err != nil { + return i, commas, indices, err + } + } +} + +// scanTagsKey scans each character in a tag key. +func scanTagsKey(buf []byte, i int) (int, error) { + // First character of the key. + if i >= len(buf) || buf[i] == ' ' || buf[i] == ',' || buf[i] == '=' { + // cpu,{'', ' ', ',', '='} + return i, fmt.Errorf("missing tag key") + } + + // Examine each character in the tag key until we hit an unescaped + // equals (the tag value), or we hit an error (i.e., unescaped + // space or comma). + for { + i++ + + // Either we reached the end of the buffer or we hit an + // unescaped comma or space. + if i >= len(buf) || + ((buf[i] == ' ' || buf[i] == ',') && buf[i-1] != '\\') { + // cpu,tag{'', ' ', ','} + return i, fmt.Errorf("missing tag value") + } + + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag= + return i + 1, nil + } + } +} + +// scanTagsValue scans each character in a tag value. +func scanTagsValue(buf []byte, i int) (int, int, error) { + // Tag value cannot be empty. + if i >= len(buf) || buf[i] == ',' || buf[i] == ' ' { + // cpu,tag={',', ' '} + return -1, i, fmt.Errorf("missing tag value") + } + + // Examine each character in the tag value until we hit an unescaped + // comma (move onto next tag key), an unescaped space (move onto + // fields), or we error out. + for { + i++ + if i >= len(buf) { + // cpu,tag=value + return -1, i, fmt.Errorf("missing fields") + } + + // An unescaped equals sign is an invalid tag value. + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag={'=', 'fo=o'} + return -1, i, fmt.Errorf("invalid tag format") + } + + if buf[i] == ',' && buf[i-1] != '\\' { + // cpu,tag=foo, + return tagKeyState, i + 1, nil + } + + // cpu,tag=foo value=1.0 + // cpu, tag=foo\= value=1.0 + if buf[i] == ' ' && buf[i-1] != '\\' { + return fieldsState, i, nil + } + } +} + +func insertionSort(l, r int, buf []byte, indices []int) { + for i := l + 1; i < r; i++ { + for j := i; j > l && less(buf, indices, j, j-1); j-- { + indices[j], indices[j-1] = indices[j-1], indices[j] + } + } +} + +func less(buf []byte, indices []int, i, j int) bool { + // This grabs the tag names for i & j, it ignores the values + _, a := scanTo(buf, indices[i], '=') + _, b := scanTo(buf, indices[j], '=') + return bytes.Compare(a, b) < 0 +} + +// scanFields scans buf, starting at i for the fields section of a point. It returns +// the ending position and the byte slice of the fields within buf. +func scanFields(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + quoted := false + + // tracks how many '=' we've seen + equals := 0 + + // tracks how many commas we've seen + commas := 0 + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // escaped characters? + if buf[i] == '\\' && i+1 < len(buf) { + i += 2 + continue + } + + // If the value is quoted, scan until we get to the end quote + // Only quote values in the field value since quotes are not significant + // in the field key + if buf[i] == '"' && equals > commas { + quoted = !quoted + i++ + continue + } + + // If we see an =, ensure that there is at least on char before and after it + if buf[i] == '=' && !quoted { + equals++ + + // check for "... =123" but allow "a\ =123" + if buf[i-1] == ' ' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "...a=123,=456" but allow "a=123,a\,=456" + if buf[i-1] == ',' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "... value=" + if i+1 >= len(buf) { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + // check for "... value=,value2=..." + if buf[i+1] == ',' || buf[i+1] == ' ' { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + if isNumeric(buf[i+1]) || buf[i+1] == '-' || buf[i+1] == 'N' || buf[i+1] == 'n' { + var err error + i, err = scanNumber(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + // If next byte is not a double-quote, the value must be a boolean + if buf[i+1] != '"' { + var err error + i, _, err = scanBoolean(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + } + + if buf[i] == ',' && !quoted { + commas++ + } + + // reached end of block? + if buf[i] == ' ' && !quoted { + break + } + i++ + } + + if quoted { + return i, buf[start:i], fmt.Errorf("unbalanced quotes") + } + + // check that all field sections had key and values (e.g. prevent "a=1,b" + if equals == 0 || commas != equals-1 { + return i, buf[start:i], fmt.Errorf("invalid field format") + } + + return i, buf[start:i], nil +} + +// scanTime scans buf, starting at i for the time section of a point. It +// returns the ending position and the byte slice of the timestamp within buf +// and and error if the timestamp is not in the correct numeric format. +func scanTime(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached end of block or trailing whitespace? + if buf[i] == '\n' || buf[i] == ' ' { + break + } + + // Handle negative timestamps + if i == start && buf[i] == '-' { + i++ + continue + } + + // Timestamps should be integers, make sure they are so we don't need + // to actually parse the timestamp until needed. + if buf[i] < '0' || buf[i] > '9' { + return i, buf[start:i], fmt.Errorf("bad timestamp") + } + i++ + } + return i, buf[start:i], nil +} + +func isNumeric(b byte) bool { + return (b >= '0' && b <= '9') || b == '.' +} + +// scanNumber returns the end position within buf, start at i after +// scanning over buf for an integer, or float. It returns an +// error if a invalid number is scanned. +func scanNumber(buf []byte, i int) (int, error) { + start := i + var isInt, isUnsigned bool + + // Is negative number? + if i < len(buf) && buf[i] == '-' { + i++ + // There must be more characters now, as just '-' is illegal. + if i == len(buf) { + return i, ErrInvalidNumber + } + } + + // how many decimal points we've see + decimal := false + + // indicates the number is float in scientific notation + scientific := false + + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + + if buf[i] == 'i' && i > start && !(isInt || isUnsigned) { + isInt = true + i++ + continue + } else if buf[i] == 'u' && i > start && !(isInt || isUnsigned) { + isUnsigned = true + i++ + continue + } + + if buf[i] == '.' { + // Can't have more than 1 decimal (e.g. 1.1.1 should fail) + if decimal { + return i, ErrInvalidNumber + } + decimal = true + } + + // `e` is valid for floats but not as the first char + if i > start && (buf[i] == 'e' || buf[i] == 'E') { + scientific = true + i++ + continue + } + + // + and - are only valid at this point if they follow an e (scientific notation) + if (buf[i] == '+' || buf[i] == '-') && (buf[i-1] == 'e' || buf[i-1] == 'E') { + i++ + continue + } + + // NaN is an unsupported value + if i+2 < len(buf) && (buf[i] == 'N' || buf[i] == 'n') { + return i, ErrInvalidNumber + } + + if !isNumeric(buf[i]) { + return i, ErrInvalidNumber + } + i++ + } + + if (isInt || isUnsigned) && (decimal || scientific) { + return i, ErrInvalidNumber + } + + numericDigits := i - start + if isInt { + numericDigits-- + } + if decimal { + numericDigits-- + } + if buf[start] == '-' { + numericDigits-- + } + + if numericDigits == 0 { + return i, ErrInvalidNumber + } + + // It's more common that numbers will be within min/max range for their type but we need to prevent + // out or range numbers from being parsed successfully. This uses some simple heuristics to decide + // if we should parse the number to the actual type. It does not do it all the time because it incurs + // extra allocations and we end up converting the type again when writing points to disk. + if isInt { + // Make sure the last char is an 'i' for integers (e.g. 9i10 is not valid) + if buf[i-1] != 'i' { + return i, ErrInvalidNumber + } + // Parse the int to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `i` from our tests + if len(buf[start:i-1]) >= maxInt64Digits || len(buf[start:i-1]) >= minInt64Digits { + if _, err := parseIntBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) + } + } + } else if isUnsigned { + // Return an error if uint64 support has not been enabled. + if !enableUint64Support { + return i, ErrInvalidNumber + } + // Make sure the last char is a 'u' for unsigned + if buf[i-1] != 'u' { + return i, ErrInvalidNumber + } + // Make sure the first char is not a '-' for unsigned + if buf[start] == '-' { + return i, ErrInvalidNumber + } + // Parse the uint to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `u` from our tests + if len(buf[start:i-1]) >= maxUint64Digits { + if _, err := parseUintBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse unsigned %s: %s", buf[start:i-1], err) + } + } + } else { + // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range + if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { + if _, err := parseFloatBytes(buf[start:i], 10); err != nil { + return i, fmt.Errorf("invalid float") + } + } + } + + return i, nil +} + +// scanBoolean returns the end position within buf, start at i after +// scanning over buf for boolean. Valid values for a boolean are +// t, T, true, TRUE, f, F, false, FALSE. It returns an error if a invalid boolean +// is scanned. +func scanBoolean(buf []byte, i int) (int, []byte, error) { + start := i + + if i < len(buf) && (buf[i] != 't' && buf[i] != 'f' && buf[i] != 'T' && buf[i] != 'F') { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + i++ + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + i++ + } + + // Single char bool (t, T, f, F) is ok + if i-start == 1 { + return i, buf[start:i], nil + } + + // length must be 4 for true or TRUE + if (buf[start] == 't' || buf[start] == 'T') && i-start != 4 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // length must be 5 for false or FALSE + if (buf[start] == 'f' || buf[start] == 'F') && i-start != 5 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // Otherwise + valid := false + switch buf[start] { + case 't': + valid = bytes.Equal(buf[start:i], []byte("true")) + case 'f': + valid = bytes.Equal(buf[start:i], []byte("false")) + case 'T': + valid = bytes.Equal(buf[start:i], []byte("TRUE")) || bytes.Equal(buf[start:i], []byte("True")) + case 'F': + valid = bytes.Equal(buf[start:i], []byte("FALSE")) || bytes.Equal(buf[start:i], []byte("False")) + } + + if !valid { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + return i, buf[start:i], nil + +} + +// skipWhitespace returns the end position within buf, starting at i after +// scanning over spaces in tags. +func skipWhitespace(buf []byte, i int) int { + for i < len(buf) { + if buf[i] != ' ' && buf[i] != '\t' && buf[i] != 0 { + break + } + i++ + } + return i +} + +// scanLine returns the end position in buf and the next line found within +// buf. +func scanLine(buf []byte, i int) (int, []byte) { + start := i + quoted := false + fields := false + + // tracks how many '=' and commas we've seen + // this duplicates some of the functionality in scanFields + equals := 0 + commas := 0 + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // skip past escaped characters + if buf[i] == '\\' && i+2 < len(buf) { + i += 2 + continue + } + + if buf[i] == ' ' { + fields = true + } + + // If we see a double quote, makes sure it is not escaped + if fields { + if !quoted && buf[i] == '=' { + i++ + equals++ + continue + } else if !quoted && buf[i] == ',' { + i++ + commas++ + continue + } else if buf[i] == '"' && equals > commas { + i++ + quoted = !quoted + continue + } + } + + if buf[i] == '\n' && !quoted { + break + } + + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte, where stop byte +// has not been escaped. +// +// If there are leading spaces, they are skipped. +func scanTo(buf []byte, i int, stop byte) (int, []byte) { + start := i + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached unescaped stop value? + if buf[i] == stop && (i == 0 || buf[i-1] != '\\') { + break + } + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte. If there are leading +// spaces, they are skipped. +func scanToSpaceOr(buf []byte, i int, stop byte) (int, []byte) { + start := i + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + + for { + i++ + if buf[i-1] == '\\' { + continue + } + + // reached the end of buf? + if i >= len(buf) { + return i, buf[start:i] + } + + // reached end of block? + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + } +} + +func scanTagValue(buf []byte, i int) (int, []byte) { + start := i + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' && buf[i-1] != '\\' { + break + } + i++ + } + if i > len(buf) { + return i, nil + } + return i, buf[start:i] +} + +func scanFieldValue(buf []byte, i int) (int, []byte) { + start := i + quoted := false + for i < len(buf) { + // Only escape char for a field value is a double-quote and backslash + if buf[i] == '\\' && i+1 < len(buf) && (buf[i+1] == '"' || buf[i+1] == '\\') { + i += 2 + continue + } + + // Quoted value? (e.g. string) + if buf[i] == '"' { + i++ + quoted = !quoted + continue + } + + if buf[i] == ',' && !quoted { + break + } + i++ + } + return i, buf[start:i] +} + +func EscapeMeasurement(in []byte) []byte { + for b, esc := range measurementEscapeCodes { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + return in +} + +func unescapeMeasurement(in []byte) []byte { + for b, esc := range measurementEscapeCodes { + in = bytes.Replace(in, esc, []byte{b}, -1) + } + return in +} + +func escapeTag(in []byte) []byte { + for b, esc := range tagEscapeCodes { + if bytes.IndexByte(in, b) != -1 { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + } + return in +} + +func unescapeTag(in []byte) []byte { + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + for b, esc := range tagEscapeCodes { + if bytes.IndexByte(in, b) != -1 { + in = bytes.Replace(in, esc, []byte{b}, -1) + } + } + return in +} + +// escapeStringFieldReplacer replaces double quotes and backslashes +// with the same character preceded by a backslash. +// As of Go 1.7 this benchmarked better in allocations and CPU time +// compared to iterating through a string byte-by-byte and appending to a new byte slice, +// calling strings.Replace twice, and better than (*Regex).ReplaceAllString. +var escapeStringFieldReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) + +// EscapeStringField returns a copy of in with any double quotes or +// backslashes with escaped values. +func EscapeStringField(in string) string { + return escapeStringFieldReplacer.Replace(in) +} + +// unescapeStringField returns a copy of in with any escaped double-quotes +// or backslashes unescaped. +func unescapeStringField(in string) string { + if strings.IndexByte(in, '\\') == -1 { + return in + } + + var out []byte + i := 0 + for { + if i >= len(in) { + break + } + // unescape backslashes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '\\' { + out = append(out, '\\') + i += 2 + continue + } + // unescape double-quotes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '"' { + out = append(out, '"') + i += 2 + continue + } + out = append(out, in[i]) + i++ + + } + return string(out) +} + +// NewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN) or out of range time is passed, this function returns an error. +func NewPoint(name string, tags Tags, fields Fields, t time.Time) (Point, error) { + key, err := pointKey(name, tags, fields, t) + if err != nil { + return nil, err + } + + return &point{ + key: key, + time: t, + fields: fields.MarshalBinary(), + }, nil +} + +// pointKey checks some basic requirements for valid points, and returns the +// key, along with an possible error. +func pointKey(measurement string, tags Tags, fields Fields, t time.Time) ([]byte, error) { + if len(fields) == 0 { + return nil, ErrPointMustHaveAField + } + + if !t.IsZero() { + if err := CheckTime(t); err != nil { + return nil, err + } + } + + for key, value := range fields { + switch value := value.(type) { + case float64: + // Ensure the caller validates and handles invalid field values + if math.IsNaN(value) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + case float32: + // Ensure the caller validates and handles invalid field values + if math.IsNaN(float64(value)) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + } + if len(key) == 0 { + return nil, fmt.Errorf("all fields must have non-empty names") + } + } + + key := MakeKey([]byte(measurement), tags) + for field := range fields { + sz := seriesKeySize(key, []byte(field)) + if sz > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + } + } + + return key, nil +} + +func seriesKeySize(key, field []byte) int { + // 4 is the length of the tsm1.fieldKeySeparator constant. It's inlined here to avoid a circular + // dependency. + return len(key) + 4 + len(field) +} + +// NewPointFromBytes returns a new Point from a marshalled Point. +func NewPointFromBytes(b []byte) (Point, error) { + p := &point{} + if err := p.UnmarshalBinary(b); err != nil { + return nil, err + } + + // This does some basic validation to ensure there are fields and they + // can be unmarshalled as well. + iter := p.FieldIterator() + var hasField bool + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + hasField = true + switch iter.Type() { + case Float: + _, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Integer: + _, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Unsigned: + _, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case String: + // Skip since this won't return an error + case Boolean: + _, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + } + } + + if !hasField { + return nil, ErrPointMustHaveAField + } + + return p, nil +} + +// MustNewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN) is passed, this function panics. +func MustNewPoint(name string, tags Tags, fields Fields, time time.Time) Point { + pt, err := NewPoint(name, tags, fields, time) + if err != nil { + panic(err.Error()) + } + return pt +} + +// Key returns the key (measurement joined with tags) of the point. +func (p *point) Key() []byte { + return p.key +} + +func (p *point) name() []byte { + _, name := scanTo(p.key, 0, ',') + return name +} + +func (p *point) Name() []byte { + return escape.Unescape(p.name()) +} + +// SetName updates the measurement name for the point. +func (p *point) SetName(name string) { + p.cachedName = "" + p.key = MakeKey([]byte(name), p.Tags()) +} + +// Time return the timestamp for the point. +func (p *point) Time() time.Time { + return p.time +} + +// SetTime updates the timestamp for the point. +func (p *point) SetTime(t time.Time) { + p.time = t +} + +// Round will round the timestamp of the point to the given duration. +func (p *point) Round(d time.Duration) { + p.time = p.time.Round(d) +} + +// Tags returns the tag set for the point. +func (p *point) Tags() Tags { + if p.cachedTags != nil { + return p.cachedTags + } + p.cachedTags = parseTags(p.key) + return p.cachedTags +} + +func (p *point) HasTag(tag []byte) bool { + if len(p.key) == 0 { + return false + } + + var exists bool + walkTags(p.key, func(key, value []byte) bool { + if bytes.Equal(tag, key) { + exists = true + return false + } + return true + }) + + return exists +} + +func walkTags(buf []byte, fn func(key, value []byte) bool) { + if len(buf) == 0 { + return + } + + pos, name := scanTo(buf, 0, ',') + + // it's an empty key, so there are no tags + if len(name) == 0 { + return + } + + hasEscape := bytes.IndexByte(buf, '\\') != -1 + i := pos + 1 + var key, value []byte + for { + if i >= len(buf) { + break + } + i, key = scanTo(buf, i, '=') + i, value = scanTagValue(buf, i+1) + + if len(value) == 0 { + continue + } + + if hasEscape { + if !fn(unescapeTag(key), unescapeTag(value)) { + return + } + } else { + if !fn(key, value) { + return + } + } + + i++ + } +} + +// walkFields walks each field key and value via fn. If fn returns false, the iteration +// is stopped. The values are the raw byte slices and not the converted types. +func walkFields(buf []byte, fn func(key, value []byte) bool) { + var i int + var key, val []byte + for len(buf) > 0 { + i, key = scanTo(buf, 0, '=') + buf = buf[i+1:] + i, val = scanFieldValue(buf, 0) + buf = buf[i:] + if !fn(key, val) { + break + } + + // slice off comma + if len(buf) > 0 { + buf = buf[1:] + } + } +} + +func parseTags(buf []byte) Tags { + if len(buf) == 0 { + return nil + } + + tags := make(Tags, bytes.Count(buf, []byte(","))) + p := 0 + walkTags(buf, func(key, value []byte) bool { + tags[p].Key = key + tags[p].Value = value + p++ + return true + }) + return tags +} + +// MakeKey creates a key for a set of tags. +func MakeKey(name []byte, tags Tags) []byte { + // unescape the name and then re-escape it to avoid double escaping. + // The key should always be stored in escaped form. + return append(EscapeMeasurement(unescapeMeasurement(name)), tags.HashKey()...) +} + +// SetTags replaces the tags for the point. +func (p *point) SetTags(tags Tags) { + p.key = MakeKey(p.Name(), tags) + p.cachedTags = tags +} + +// AddTag adds or replaces a tag value for a point. +func (p *point) AddTag(key, value string) { + tags := p.Tags() + tags = append(tags, Tag{Key: []byte(key), Value: []byte(value)}) + sort.Sort(tags) + p.cachedTags = tags + p.key = MakeKey(p.Name(), tags) +} + +// Fields returns the fields for the point. +func (p *point) Fields() (Fields, error) { + if p.cachedFields != nil { + return p.cachedFields, nil + } + cf, err := p.unmarshalBinary() + if err != nil { + return nil, err + } + p.cachedFields = cf + return p.cachedFields, nil +} + +// SetPrecision will round a time to the specified precision. +func (p *point) SetPrecision(precision string) { + switch precision { + case "n": + case "u": + p.SetTime(p.Time().Truncate(time.Microsecond)) + case "ms": + p.SetTime(p.Time().Truncate(time.Millisecond)) + case "s": + p.SetTime(p.Time().Truncate(time.Second)) + case "m": + p.SetTime(p.Time().Truncate(time.Minute)) + case "h": + p.SetTime(p.Time().Truncate(time.Hour)) + } +} + +// String returns the string representation of the point. +func (p *point) String() string { + if p.Time().IsZero() { + return string(p.Key()) + " " + string(p.fields) + } + return string(p.Key()) + " " + string(p.fields) + " " + strconv.FormatInt(p.UnixNano(), 10) +} + +// AppendString appends the string representation of the point to buf. +func (p *point) AppendString(buf []byte) []byte { + buf = append(buf, p.key...) + buf = append(buf, ' ') + buf = append(buf, p.fields...) + + if !p.time.IsZero() { + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, p.UnixNano(), 10) + } + + return buf +} + +// StringSize returns the length of the string that would be returned by String(). +func (p *point) StringSize() int { + size := len(p.key) + len(p.fields) + 1 + + if !p.time.IsZero() { + digits := 1 // even "0" has one digit + t := p.UnixNano() + if t < 0 { + // account for negative sign, then negate + digits++ + t = -t + } + for t > 9 { // already accounted for one digit + digits++ + t /= 10 + } + size += digits + 1 // digits and a space + } + + return size +} + +// MarshalBinary returns a binary representation of the point. +func (p *point) MarshalBinary() ([]byte, error) { + if len(p.fields) == 0 { + return nil, ErrPointMustHaveAField + } + + tb, err := p.time.MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 8+len(p.key)+len(p.fields)+len(tb)) + i := 0 + + binary.BigEndian.PutUint32(b[i:], uint32(len(p.key))) + i += 4 + + i += copy(b[i:], p.key) + + binary.BigEndian.PutUint32(b[i:i+4], uint32(len(p.fields))) + i += 4 + + i += copy(b[i:], p.fields) + + copy(b[i:], tb) + return b, nil +} + +// UnmarshalBinary decodes a binary representation of the point into a point struct. +func (p *point) UnmarshalBinary(b []byte) error { + var n int + + // Read key length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read key. + if len(b) < n { + return io.ErrShortBuffer + } + p.key, b = b[:n], b[n:] + + // Read fields length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read fields. + if len(b) < n { + return io.ErrShortBuffer + } + p.fields, b = b[:n], b[n:] + + // Read timestamp. + if err := p.time.UnmarshalBinary(b); err != nil { + return err + } + return nil +} + +// PrecisionString returns a string representation of the point. If there +// is a timestamp associated with the point then it will be specified in the +// given unit. +func (p *point) PrecisionString(precision string) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.UnixNano()/GetPrecisionMultiplier(precision)) +} + +// RoundedString returns a string representation of the point. If there +// is a timestamp associated with the point, then it will be rounded to the +// given duration. +func (p *point) RoundedString(d time.Duration) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.time.Round(d).UnixNano()) +} + +func (p *point) unmarshalBinary() (Fields, error) { + iter := p.FieldIterator() + fields := make(Fields, 8) + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + switch iter.Type() { + case Float: + v, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Integer: + v, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Unsigned: + v, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case String: + fields[string(iter.FieldKey())] = iter.StringValue() + case Boolean: + v, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + } + } + return fields, nil +} + +// HashID returns a non-cryptographic checksum of the point's key. +func (p *point) HashID() uint64 { + h := NewInlineFNV64a() + h.Write(p.key) + sum := h.Sum64() + return sum +} + +// UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. +func (p *point) UnixNano() int64 { + return p.Time().UnixNano() +} + +// Split will attempt to return multiple points with the same timestamp whose +// string representations are no longer than size. Points with a single field or +// a point without a timestamp may exceed the requested size. +func (p *point) Split(size int) []Point { + if p.time.IsZero() || p.StringSize() <= size { + return []Point{p} + } + + // key string, timestamp string, spaces + size -= len(p.key) + len(strconv.FormatInt(p.time.UnixNano(), 10)) + 2 + + var points []Point + var start, cur int + + for cur < len(p.fields) { + end, _ := scanTo(p.fields, cur, '=') + end, _ = scanFieldValue(p.fields, end+1) + + if cur > start && end-start > size { + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start : cur-1], + }) + start = cur + } + + cur = end + 1 + } + + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start:], + }) + + return points +} + +// Tag represents a single key/value tag pair. +type Tag struct { + Key []byte + Value []byte +} + +// NewTag returns a new Tag. +func NewTag(key, value []byte) Tag { + return Tag{ + Key: key, + Value: value, + } +} + +// Size returns the size of the key and value. +func (t Tag) Size() int { return len(t.Key) + len(t.Value) } + +// Clone returns a shallow copy of Tag. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (t Tag) Clone() Tag { + other := Tag{ + Key: make([]byte, len(t.Key)), + Value: make([]byte, len(t.Value)), + } + + copy(other.Key, t.Key) + copy(other.Value, t.Value) + + return other +} + +// String returns the string reprsentation of the tag. +func (t *Tag) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + buf.WriteString(string(t.Key)) + buf.WriteByte(' ') + buf.WriteString(string(t.Value)) + buf.WriteByte('}') + return buf.String() +} + +// Tags represents a sorted list of tags. +type Tags []Tag + +// NewTags returns a new Tags from a map. +func NewTags(m map[string]string) Tags { + if len(m) == 0 { + return nil + } + a := make(Tags, 0, len(m)) + for k, v := range m { + a = append(a, NewTag([]byte(k), []byte(v))) + } + sort.Sort(a) + return a +} + +// Keys returns the list of keys for a tag set. +func (a Tags) Keys() []string { + if len(a) == 0 { + return nil + } + keys := make([]string, len(a)) + for i, tag := range a { + keys[i] = string(tag.Key) + } + return keys +} + +// Values returns the list of values for a tag set. +func (a Tags) Values() []string { + if len(a) == 0 { + return nil + } + values := make([]string, len(a)) + for i, tag := range a { + values[i] = string(tag.Value) + } + return values +} + +// String returns the string representation of the tags. +func (a Tags) String() string { + var buf bytes.Buffer + buf.WriteByte('[') + for i := range a { + buf.WriteString(a[i].String()) + if i < len(a)-1 { + buf.WriteByte(' ') + } + } + buf.WriteByte(']') + return buf.String() +} + +// Size returns the number of bytes needed to store all tags. Note, this is +// the number of bytes needed to store all keys and values and does not account +// for data structures or delimiters for example. +func (a Tags) Size() int { + var total int + for _, t := range a { + total += t.Size() + } + return total +} + +// Clone returns a copy of the slice where the elements are a result of calling `Clone` on the original elements +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (a Tags) Clone() Tags { + if len(a) == 0 { + return nil + } + + others := make(Tags, len(a)) + for i := range a { + others[i] = a[i].Clone() + } + + return others +} + +func (a Tags) Len() int { return len(a) } +func (a Tags) Less(i, j int) bool { return bytes.Compare(a[i].Key, a[j].Key) == -1 } +func (a Tags) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// Equal returns true if a equals other. +func (a Tags) Equal(other Tags) bool { + if len(a) != len(other) { + return false + } + for i := range a { + if !bytes.Equal(a[i].Key, other[i].Key) || !bytes.Equal(a[i].Value, other[i].Value) { + return false + } + } + return true +} + +// CompareTags returns -1 if a < b, 1 if a > b, and 0 if a == b. +func CompareTags(a, b Tags) int { + // Compare each key & value until a mismatch. + for i := 0; i < len(a) && i < len(b); i++ { + if cmp := bytes.Compare(a[i].Key, b[i].Key); cmp != 0 { + return cmp + } + if cmp := bytes.Compare(a[i].Value, b[i].Value); cmp != 0 { + return cmp + } + } + + // If all tags are equal up to this point then return shorter tagset. + if len(a) < len(b) { + return -1 + } else if len(a) > len(b) { + return 1 + } + + // All tags are equal. + return 0 +} + +// Get returns the value for a key. +func (a Tags) Get(key []byte) []byte { + // OPTIMIZE: Use sort.Search if tagset is large. + + for _, t := range a { + if bytes.Equal(t.Key, key) { + return t.Value + } + } + return nil +} + +// GetString returns the string value for a string key. +func (a Tags) GetString(key string) string { + return string(a.Get([]byte(key))) +} + +// Set sets the value for a key. +func (a *Tags) Set(key, value []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + (*a)[i].Value = value + return + } + } + *a = append(*a, Tag{Key: key, Value: value}) + sort.Sort(*a) +} + +// SetString sets the string value for a string key. +func (a *Tags) SetString(key, value string) { + a.Set([]byte(key), []byte(value)) +} + +// Delete removes a tag by key. +func (a *Tags) Delete(key []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + copy((*a)[i:], (*a)[i+1:]) + (*a)[len(*a)-1] = Tag{} + *a = (*a)[:len(*a)-1] + return + } + } +} + +// Map returns a map representation of the tags. +func (a Tags) Map() map[string]string { + m := make(map[string]string, len(a)) + for _, t := range a { + m[string(t.Key)] = string(t.Value) + } + return m +} + +// Merge merges the tags combining the two. If both define a tag with the +// same key, the merged value overwrites the old value. +// A new map is returned. +func (a Tags) Merge(other map[string]string) Tags { + merged := make(map[string]string, len(a)+len(other)) + for _, t := range a { + merged[string(t.Key)] = string(t.Value) + } + for k, v := range other { + merged[k] = v + } + return NewTags(merged) +} + +// HashKey hashes all of a tag's keys. +func (a Tags) HashKey() []byte { + // Empty maps marshal to empty bytes. + if len(a) == 0 { + return nil + } + + // Type invariant: Tags are sorted + + escaped := make(Tags, 0, len(a)) + sz := 0 + for _, t := range a { + ek := escapeTag(t.Key) + ev := escapeTag(t.Value) + + if len(ev) > 0 { + escaped = append(escaped, Tag{Key: ek, Value: ev}) + sz += len(ek) + len(ev) + } + } + + sz += len(escaped) + (len(escaped) * 2) // separators + + // Generate marshaled bytes. + b := make([]byte, sz) + buf := b + idx := 0 + for _, k := range escaped { + buf[idx] = ',' + idx++ + copy(buf[idx:idx+len(k.Key)], k.Key) + idx += len(k.Key) + buf[idx] = '=' + idx++ + copy(buf[idx:idx+len(k.Value)], k.Value) + idx += len(k.Value) + } + return b[:idx] +} + +// CopyTags returns a shallow copy of tags. +func CopyTags(a Tags) Tags { + other := make(Tags, len(a)) + copy(other, a) + return other +} + +// DeepCopyTags returns a deep copy of tags. +func DeepCopyTags(a Tags) Tags { + // Calculate size of keys/values in bytes. + var n int + for _, t := range a { + n += len(t.Key) + len(t.Value) + } + + // Build single allocation for all key/values. + buf := make([]byte, n) + + // Copy tags to new set. + other := make(Tags, len(a)) + for i, t := range a { + copy(buf, t.Key) + other[i].Key, buf = buf[:len(t.Key)], buf[len(t.Key):] + + copy(buf, t.Value) + other[i].Value, buf = buf[:len(t.Value)], buf[len(t.Value):] + } + + return other +} + +// Fields represents a mapping between a Point's field names and their +// values. +type Fields map[string]interface{} + +// FieldIterator retuns a FieldIterator that can be used to traverse the +// fields of a point without constructing the in-memory map. +func (p *point) FieldIterator() FieldIterator { + p.Reset() + return p +} + +type fieldIterator struct { + start, end int + key, keybuf []byte + valueBuf []byte + fieldType FieldType +} + +// Next indicates whether there any fields remaining. +func (p *point) Next() bool { + p.it.start = p.it.end + if p.it.start >= len(p.fields) { + return false + } + + p.it.end, p.it.key = scanTo(p.fields, p.it.start, '=') + if escape.IsEscaped(p.it.key) { + p.it.keybuf = escape.AppendUnescaped(p.it.keybuf[:0], p.it.key) + p.it.key = p.it.keybuf + } + + p.it.end, p.it.valueBuf = scanFieldValue(p.fields, p.it.end+1) + p.it.end++ + + if len(p.it.valueBuf) == 0 { + p.it.fieldType = Empty + return true + } + + c := p.it.valueBuf[0] + + if c == '"' { + p.it.fieldType = String + return true + } + + if strings.IndexByte(`0123456789-.nNiIu`, c) >= 0 { + if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { + p.it.fieldType = Integer + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else if p.it.valueBuf[len(p.it.valueBuf)-1] == 'u' { + p.it.fieldType = Unsigned + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else { + p.it.fieldType = Float + } + return true + } + + // to keep the same behavior that currently exists, default to boolean + p.it.fieldType = Boolean + return true +} + +// FieldKey returns the key of the current field. +func (p *point) FieldKey() []byte { + return p.it.key +} + +// Type returns the FieldType of the current field. +func (p *point) Type() FieldType { + return p.it.fieldType +} + +// StringValue returns the string value of the current field. +func (p *point) StringValue() string { + return unescapeStringField(string(p.it.valueBuf[1 : len(p.it.valueBuf)-1])) +} + +// IntegerValue returns the integer value of the current field. +func (p *point) IntegerValue() (int64, error) { + n, err := parseIntBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse integer value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// UnsignedValue returns the unsigned value of the current field. +func (p *point) UnsignedValue() (uint64, error) { + n, err := parseUintBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse unsigned value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// BooleanValue returns the boolean value of the current field. +func (p *point) BooleanValue() (bool, error) { + b, err := parseBoolBytes(p.it.valueBuf) + if err != nil { + return false, fmt.Errorf("unable to parse bool value %q: %v", p.it.valueBuf, err) + } + return b, nil +} + +// FloatValue returns the float value of the current field. +func (p *point) FloatValue() (float64, error) { + f, err := parseFloatBytes(p.it.valueBuf, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse floating point value %q: %v", p.it.valueBuf, err) + } + return f, nil +} + +// Reset resets the iterator to its initial state. +func (p *point) Reset() { + p.it.fieldType = Empty + p.it.key = nil + p.it.valueBuf = nil + p.it.start = 0 + p.it.end = 0 +} + +// MarshalBinary encodes all the fields to their proper type and returns the binary +// represenation +// NOTE: uint64 is specifically not supported due to potential overflow when we decode +// again later to an int64 +// NOTE2: uint is accepted, and may be 64 bits, and is for some reason accepted... +func (p Fields) MarshalBinary() []byte { + var b []byte + keys := make([]string, 0, len(p)) + + for k := range p { + keys = append(keys, k) + } + + // Not really necessary, can probably be removed. + sort.Strings(keys) + + for i, k := range keys { + if i > 0 { + b = append(b, ',') + } + b = appendField(b, k, p[k]) + } + + return b +} + +func appendField(b []byte, k string, v interface{}) []byte { + b = append(b, []byte(escape.String(k))...) + b = append(b, '=') + + // check popular types first + switch v := v.(type) { + case float64: + b = strconv.AppendFloat(b, v, 'f', -1, 64) + case int64: + b = strconv.AppendInt(b, v, 10) + b = append(b, 'i') + case string: + b = append(b, '"') + b = append(b, []byte(EscapeStringField(v))...) + b = append(b, '"') + case bool: + b = strconv.AppendBool(b, v) + case int32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint64: + b = strconv.AppendUint(b, v, 10) + b = append(b, 'u') + case uint32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint: + // TODO: 'uint' should be converted to writing as an unsigned integer, + // but we cannot since that would break backwards compatibility. + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case float32: + b = strconv.AppendFloat(b, float64(v), 'f', -1, 32) + case []byte: + b = append(b, v...) + case nil: + // skip + default: + // Can't determine the type, so convert to string + b = append(b, '"') + b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...) + b = append(b, '"') + + } + + return b +} + +type byteSlices [][]byte + +func (a byteSlices) Len() int { return len(a) } +func (a byteSlices) Less(i, j int) bool { return bytes.Compare(a[i], a[j]) == -1 } +func (a byteSlices) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/vendor/github.com/influxdata/influxdb/models/rows.go b/vendor/github.com/influxdata/influxdb/models/rows.go new file mode 100644 index 0000000000..c087a4882d --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/rows.go @@ -0,0 +1,62 @@ +package models + +import ( + "sort" +) + +// Row represents a single row returned from the execution of a statement. +type Row struct { + Name string `json:"name,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Columns []string `json:"columns,omitempty"` + Values [][]interface{} `json:"values,omitempty"` + Partial bool `json:"partial,omitempty"` +} + +// SameSeries returns true if r contains values for the same series as o. +func (r *Row) SameSeries(o *Row) bool { + return r.tagsHash() == o.tagsHash() && r.Name == o.Name +} + +// tagsHash returns a hash of tag key/value pairs. +func (r *Row) tagsHash() uint64 { + h := NewInlineFNV64a() + keys := r.tagsKeys() + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte(r.Tags[k])) + } + return h.Sum64() +} + +// tagKeys returns a sorted list of tag keys. +func (r *Row) tagsKeys() []string { + a := make([]string, 0, len(r.Tags)) + for k := range r.Tags { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// Rows represents a collection of rows. Rows implements sort.Interface. +type Rows []*Row + +// Len implements sort.Interface. +func (p Rows) Len() int { return len(p) } + +// Less implements sort.Interface. +func (p Rows) Less(i, j int) bool { + // Sort by name first. + if p[i].Name != p[j].Name { + return p[i].Name < p[j].Name + } + + // Sort by tag set hash. Tags don't have a meaningful sort order so we + // just compute a hash and sort by that instead. This allows the tests + // to receive rows in a predictable order every time. + return p[i].tagsHash() < p[j].tagsHash() +} + +// Swap implements sort.Interface. +func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/influxdata/influxdb/models/statistic.go b/vendor/github.com/influxdata/influxdb/models/statistic.go new file mode 100644 index 0000000000..553e9d09fb --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/statistic.go @@ -0,0 +1,42 @@ +package models + +// Statistic is the representation of a statistic used by the monitoring service. +type Statistic struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + Values map[string]interface{} `json:"values"` +} + +// NewStatistic returns an initialized Statistic. +func NewStatistic(name string) Statistic { + return Statistic{ + Name: name, + Tags: make(map[string]string), + Values: make(map[string]interface{}), + } +} + +// StatisticTags is a map that can be merged with others without causing +// mutations to either map. +type StatisticTags map[string]string + +// Merge creates a new map containing the merged contents of tags and t. +// If both tags and the receiver map contain the same key, the value in tags +// is used in the resulting map. +// +// Merge always returns a usable map. +func (t StatisticTags) Merge(tags map[string]string) map[string]string { + // Add everything in tags to the result. + out := make(map[string]string, len(tags)) + for k, v := range tags { + out[k] = v + } + + // Only add values from t that don't appear in tags. + for k, v := range t { + if _, ok := tags[k]; !ok { + out[k] = v + } + } + return out +} diff --git a/vendor/github.com/influxdata/influxdb/models/time.go b/vendor/github.com/influxdata/influxdb/models/time.go new file mode 100644 index 0000000000..e98f2cb336 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/time.go @@ -0,0 +1,74 @@ +package models + +// Helper time methods since parsing time can easily overflow and we only support a +// specific time range. + +import ( + "fmt" + "math" + "time" +) + +const ( + // MinNanoTime is the minumum time that can be represented. + // + // 1677-09-21 00:12:43.145224194 +0000 UTC + // + // The two lowest minimum integers are used as sentinel values. The + // minimum value needs to be used as a value lower than any other value for + // comparisons and another separate value is needed to act as a sentinel + // default value that is unusable by the user, but usable internally. + // Because these two values need to be used for a special purpose, we do + // not allow users to write points at these two times. + MinNanoTime = int64(math.MinInt64) + 2 + + // MaxNanoTime is the maximum time that can be represented. + // + // 2262-04-11 23:47:16.854775806 +0000 UTC + // + // The highest time represented by a nanosecond needs to be used for an + // exclusive range in the shard group, so the maximum time needs to be one + // less than the possible maximum number of nanoseconds representable by an + // int64 so that we don't lose a point at that one time. + MaxNanoTime = int64(math.MaxInt64) - 1 +) + +var ( + minNanoTime = time.Unix(0, MinNanoTime).UTC() + maxNanoTime = time.Unix(0, MaxNanoTime).UTC() + + // ErrTimeOutOfRange gets returned when time is out of the representable range using int64 nanoseconds since the epoch. + ErrTimeOutOfRange = fmt.Errorf("time outside range %d - %d", MinNanoTime, MaxNanoTime) +) + +// SafeCalcTime safely calculates the time given. Will return error if the time is outside the +// supported range. +func SafeCalcTime(timestamp int64, precision string) (time.Time, error) { + mult := GetPrecisionMultiplier(precision) + if t, ok := safeSignedMult(timestamp, mult); ok { + tme := time.Unix(0, t).UTC() + return tme, CheckTime(tme) + } + + return time.Time{}, ErrTimeOutOfRange +} + +// CheckTime checks that a time is within the safe range. +func CheckTime(t time.Time) error { + if t.Before(minNanoTime) || t.After(maxNanoTime) { + return ErrTimeOutOfRange + } + return nil +} + +// Perform the multiplication and check to make sure it didn't overflow. +func safeSignedMult(a, b int64) (int64, bool) { + if a == 0 || b == 0 || a == 1 || b == 1 { + return a * b, true + } + if a == MinNanoTime || b == MaxNanoTime { + return 0, false + } + c := a * b + return c, c/b == a +} diff --git a/vendor/github.com/influxdata/influxdb/models/uint_support.go b/vendor/github.com/influxdata/influxdb/models/uint_support.go new file mode 100644 index 0000000000..18d1ca06e2 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/uint_support.go @@ -0,0 +1,7 @@ +// +build uint uint64 + +package models + +func init() { + EnableUintSupport() +} diff --git a/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go b/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go new file mode 100644 index 0000000000..f3b31f42d3 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go @@ -0,0 +1,115 @@ +// Package escape contains utilities for escaping parts of InfluxQL +// and InfluxDB line protocol. +package escape // import "github.com/influxdata/influxdb/pkg/escape" + +import ( + "bytes" + "strings" +) + +// Codes is a map of bytes to be escaped. +var Codes = map[byte][]byte{ + ',': []byte(`\,`), + '"': []byte(`\"`), + ' ': []byte(`\ `), + '=': []byte(`\=`), +} + +// Bytes escapes characters on the input slice, as defined by Codes. +func Bytes(in []byte) []byte { + for b, esc := range Codes { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + return in +} + +const escapeChars = `," =` + +// IsEscaped returns whether b has any escaped characters, +// i.e. whether b seems to have been processed by Bytes. +func IsEscaped(b []byte) bool { + for len(b) > 0 { + i := bytes.IndexByte(b, '\\') + if i < 0 { + return false + } + + if i+1 < len(b) && strings.IndexByte(escapeChars, b[i+1]) >= 0 { + return true + } + b = b[i+1:] + } + return false +} + +// AppendUnescaped appends the unescaped version of src to dst +// and returns the resulting slice. +func AppendUnescaped(dst, src []byte) []byte { + var pos int + for len(src) > 0 { + next := bytes.IndexByte(src[pos:], '\\') + if next < 0 || pos+next+1 >= len(src) { + return append(dst, src...) + } + + if pos+next+1 < len(src) && strings.IndexByte(escapeChars, src[pos+next+1]) >= 0 { + if pos+next > 0 { + dst = append(dst, src[:pos+next]...) + } + src = src[pos+next+1:] + pos = 0 + } else { + pos += next + 1 + } + } + + return dst +} + +// Unescape returns a new slice containing the unescaped version of in. +func Unescape(in []byte) []byte { + if len(in) == 0 { + return nil + } + + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + i := 0 + inLen := len(in) + + // The output size will be no more than inLen. Preallocating the + // capacity of the output is faster and uses less memory than + // letting append() do its own (over)allocation. + out := make([]byte, 0, inLen) + + for { + if i >= inLen { + break + } + if in[i] == '\\' && i+1 < inLen { + switch in[i+1] { + case ',': + out = append(out, ',') + i += 2 + continue + case '"': + out = append(out, '"') + i += 2 + continue + case ' ': + out = append(out, ' ') + i += 2 + continue + case '=': + out = append(out, '=') + i += 2 + continue + } + } + out = append(out, in[i]) + i += 1 + } + return out +} diff --git a/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go b/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go new file mode 100644 index 0000000000..db98033b0d --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go @@ -0,0 +1,21 @@ +package escape + +import "strings" + +var ( + escaper = strings.NewReplacer(`,`, `\,`, `"`, `\"`, ` `, `\ `, `=`, `\=`) + unescaper = strings.NewReplacer(`\,`, `,`, `\"`, `"`, `\ `, ` `, `\=`, `=`) +) + +// UnescapeString returns unescaped version of in. +func UnescapeString(in string) string { + if strings.IndexByte(in, '\\') == -1 { + return in + } + return unescaper.Replace(in) +} + +// String returns the escaped version of in. +func String(in string) string { + return escaper.Replace(in) +} diff --git a/vendor/github.com/rcrowley/go-metrics/json.go b/vendor/github.com/rcrowley/go-metrics/json.go deleted file mode 100644 index 2fdcbcfbf1..0000000000 --- a/vendor/github.com/rcrowley/go-metrics/json.go +++ /dev/null @@ -1,87 +0,0 @@ -package metrics - -import ( - "encoding/json" - "io" - "time" -) - -// MarshalJSON returns a byte slice containing a JSON representation of all -// the metrics in the Registry. -func (r *StandardRegistry) MarshalJSON() ([]byte, error) { - data := make(map[string]map[string]interface{}) - r.Each(func(name string, i interface{}) { - values := make(map[string]interface{}) - switch metric := i.(type) { - case Counter: - values["count"] = metric.Count() - case Gauge: - values["value"] = metric.Value() - case GaugeFloat64: - values["value"] = metric.Value() - case Healthcheck: - values["error"] = nil - metric.Check() - if err := metric.Error(); nil != err { - values["error"] = metric.Error().Error() - } - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = h.Count() - values["min"] = h.Min() - values["max"] = h.Max() - values["mean"] = h.Mean() - values["stddev"] = h.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - case Meter: - m := metric.Snapshot() - values["count"] = m.Count() - values["1m.rate"] = m.Rate1() - values["5m.rate"] = m.Rate5() - values["15m.rate"] = m.Rate15() - values["mean.rate"] = m.RateMean() - case Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = t.Count() - values["min"] = t.Min() - values["max"] = t.Max() - values["mean"] = t.Mean() - values["stddev"] = t.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - values["1m.rate"] = t.Rate1() - values["5m.rate"] = t.Rate5() - values["15m.rate"] = t.Rate15() - values["mean.rate"] = t.RateMean() - } - data[name] = values - }) - return json.Marshal(data) -} - -// WriteJSON writes metrics from the given registry periodically to the -// specified io.Writer as JSON. -func WriteJSON(r Registry, d time.Duration, w io.Writer) { - for _ = range time.Tick(d) { - WriteJSONOnce(r, w) - } -} - -// WriteJSONOnce writes metrics from the given registry to the specified -// io.Writer as JSON. -func WriteJSONOnce(r Registry, w io.Writer) { - json.NewEncoder(w).Encode(r) -} - -func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) { - return json.Marshal(p.underlying) -} diff --git a/vendor/github.com/rcrowley/go-metrics/metrics.go b/vendor/github.com/rcrowley/go-metrics/metrics.go deleted file mode 100644 index b97a49ed12..0000000000 --- a/vendor/github.com/rcrowley/go-metrics/metrics.go +++ /dev/null @@ -1,13 +0,0 @@ -// Go port of Coda Hale's Metrics library -// -// -// -// Coda Hale's original work: -package metrics - -// UseNilMetrics is checked by the constructor functions for all of the -// standard metrics. If it is true, the metric returned is a stub. -// -// This global kill-switch helps quantify the observer effect and makes -// for less cluttered pprof profiles. -var UseNilMetrics bool = false diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 2248a1b129..a2b332a2e0 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -162,8 +162,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) + path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index 0edd3782f5..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.10 - -package notify - -const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go deleted file mode 100644 index b81c3c1859..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.10 - -package notify - -/* -#include -*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/vendor.json b/vendor/vendor.json index 4360223298..1341589952 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -206,6 +206,30 @@ "revision": "679507af18f3c7ba2bcc7905392ce23e148661c3", "revisionTime": "2016-12-24T10:41:01Z" }, + { + "checksumSHA1": "6tNwbL5tUS0dxYzADKVZtI2d/lE=", + "path": "github.com/influxdata/influxdb/client", + "revision": "a55dd0f50edd14c9c798d3564189eb4f53914309", + "revisionTime": "2017-10-09T17:24:46Z" + }, + { + "checksumSHA1": "O4XpbSNeUhSIMD2FWtQximJiFIs=", + "path": "github.com/influxdata/influxdb/client/v2", + "revision": "b36b9f109f2da91c8941679caf5356e08eee0b2b", + "revisionTime": "2018-01-17T01:42:09Z" + }, + { + "checksumSHA1": "cfumoC9gHEUROd+fA8qK3WLFAZQ=", + "path": "github.com/influxdata/influxdb/models", + "revision": "b36b9f109f2da91c8941679caf5356e08eee0b2b", + "revisionTime": "2018-01-17T01:42:09Z" + }, + { + "checksumSHA1": "Z0Bb5PWa5WL/j5Dm2KJCLGn1l7U=", + "path": "github.com/influxdata/influxdb/pkg/escape", + "revision": "01288bdb0883a01cac999326bd34421b29acaec8", + "revisionTime": "2018-02-21T22:33:40Z" + }, { "checksumSHA1": "vTGKMIfiMwz43y5bsgx9PrL+AVw=", "path": "github.com/jackpal/go-nat-pmp", @@ -310,22 +334,10 @@ "revisionTime": "2017-08-14T17:01:13Z" }, { - "checksumSHA1": "KAzbLjI9MzW2tjfcAsK75lVRp6I=", - "path": "github.com/rcrowley/go-metrics", - "revision": "1f30fe9094a513ce4c700b9a54458bbb0c96996c", - "revisionTime": "2016-11-28T21:05:44Z" - }, - { - "checksumSHA1": "q/d9nXRQYKEJ/EWn+5y6jL8rPGs=", - "path": "github.com/rcrowley/go-metrics/exp", - "revision": "1f30fe9094a513ce4c700b9a54458bbb0c96996c", - "revisionTime": "2016-11-28T21:05:44Z" - }, - { - "checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=", + "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", "path": "github.com/rjeczalik/notify", - "revision": "27b537f07230b3f917421af6dcf044038dbe57e2", - "revisionTime": "2018-01-03T13:19:05Z" + "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", + "revisionTime": "2018-02-03T14:01:15Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 0ec6ec570c..a889c2bd23 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -17,7 +17,6 @@ package mailserver import ( - "bytes" "encoding/binary" "fmt" @@ -26,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -175,7 +174,10 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) if len(src)-len(peerID) == 1 { src = src[1:] } - if !bytes.Equal(peerID, src) { + + // if you want to check the signature, you can do it here. e.g.: + // if !bytes.Equal(peerID, src) { + if src == nil { log.Warn(fmt.Sprintf("Wrong signature of p2p request")) return false, 0, 0, topic } diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 9155ee85a6..63864c1e6b 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) const powRequirement = 0.00001 @@ -167,8 +167,9 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * src[0]++ ok, lower, upper, topic = server.validateRequest(src, request) - if ok { - t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper) + if !ok { + // request should be valid regardless of signature + t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper) } } diff --git a/whisper/whisperv2/api.go b/whisper/whisperv2/api.go deleted file mode 100644 index 5c6d170959..0000000000 --- a/whisper/whisperv2/api.go +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" -) - -// PublicWhisperAPI provides the whisper RPC service. -type PublicWhisperAPI struct { - w *Whisper - - messagesMu sync.RWMutex - messages map[hexutil.Uint]*whisperFilter -} - -type whisperOfflineError struct{} - -func (e *whisperOfflineError) Error() string { - return "whisper is offline" -} - -// whisperOffLineErr is returned when the node doesn't offer the shh service. -var whisperOffLineErr = new(whisperOfflineError) - -// NewPublicWhisperAPI create a new RPC whisper service. -func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { - return &PublicWhisperAPI{w: w, messages: make(map[hexutil.Uint]*whisperFilter)} -} - -// Version returns the Whisper version this node offers. -func (s *PublicWhisperAPI) Version() (hexutil.Uint, error) { - if s.w == nil { - return 0, whisperOffLineErr - } - return hexutil.Uint(s.w.Version()), nil -} - -// HasIdentity checks if the the whisper node is configured with the private key -// of the specified public pair. -func (s *PublicWhisperAPI) HasIdentity(identity string) (bool, error) { - if s.w == nil { - return false, whisperOffLineErr - } - return s.w.HasIdentity(crypto.ToECDSAPub(common.FromHex(identity))), nil -} - -// NewIdentity generates a new cryptographic identity for the client, and injects -// it into the known identities for message decryption. -func (s *PublicWhisperAPI) NewIdentity() (string, error) { - if s.w == nil { - return "", whisperOffLineErr - } - - identity := s.w.NewIdentity() - return common.ToHex(crypto.FromECDSAPub(&identity.PublicKey)), nil -} - -type NewFilterArgs struct { - To string - From string - Topics [][][]byte -} - -// NewWhisperFilter creates and registers a new message filter to watch for inbound whisper messages. -func (s *PublicWhisperAPI) NewFilter(args NewFilterArgs) (hexutil.Uint, error) { - if s.w == nil { - return 0, whisperOffLineErr - } - - var id hexutil.Uint - filter := Filter{ - To: crypto.ToECDSAPub(common.FromHex(args.To)), - From: crypto.ToECDSAPub(common.FromHex(args.From)), - Topics: NewFilterTopics(args.Topics...), - Fn: func(message *Message) { - wmsg := NewWhisperMessage(message) - s.messagesMu.RLock() // Only read lock to the filter pool - defer s.messagesMu.RUnlock() - if s.messages[id] != nil { - s.messages[id].insert(wmsg) - } - }, - } - id = hexutil.Uint(s.w.Watch(filter)) - - s.messagesMu.Lock() - s.messages[id] = newWhisperFilter(id, s.w) - s.messagesMu.Unlock() - - return id, nil -} - -// GetFilterChanges retrieves all the new messages matched by a filter since the last retrieval. -func (s *PublicWhisperAPI) GetFilterChanges(filterId hexutil.Uint) []WhisperMessage { - s.messagesMu.RLock() - defer s.messagesMu.RUnlock() - - if s.messages[filterId] != nil { - if changes := s.messages[filterId].retrieve(); changes != nil { - return changes - } - } - return returnWhisperMessages(nil) -} - -// UninstallFilter disables and removes an existing filter. -func (s *PublicWhisperAPI) UninstallFilter(filterId hexutil.Uint) bool { - s.messagesMu.Lock() - defer s.messagesMu.Unlock() - - if _, ok := s.messages[filterId]; ok { - delete(s.messages, filterId) - return true - } - return false -} - -// GetMessages retrieves all the known messages that match a specific filter. -func (s *PublicWhisperAPI) GetMessages(filterId hexutil.Uint) []WhisperMessage { - // Retrieve all the cached messages matching a specific, existing filter - s.messagesMu.RLock() - defer s.messagesMu.RUnlock() - - var messages []*Message - if s.messages[filterId] != nil { - messages = s.messages[filterId].messages() - } - - return returnWhisperMessages(messages) -} - -// returnWhisperMessages converts aNhisper message to a RPC whisper message. -func returnWhisperMessages(messages []*Message) []WhisperMessage { - msgs := make([]WhisperMessage, len(messages)) - for i, msg := range messages { - msgs[i] = NewWhisperMessage(msg) - } - return msgs -} - -type PostArgs struct { - From string `json:"from"` - To string `json:"to"` - Topics [][]byte `json:"topics"` - Payload string `json:"payload"` - Priority int64 `json:"priority"` - TTL int64 `json:"ttl"` -} - -// Post injects a message into the whisper network for distribution. -func (s *PublicWhisperAPI) Post(args PostArgs) (bool, error) { - if s.w == nil { - return false, whisperOffLineErr - } - - // construct whisper message with transmission options - message := NewMessage(common.FromHex(args.Payload)) - options := Options{ - To: crypto.ToECDSAPub(common.FromHex(args.To)), - TTL: time.Duration(args.TTL) * time.Second, - Topics: NewTopics(args.Topics...), - } - - // set sender identity - if len(args.From) > 0 { - if key := s.w.GetIdentity(crypto.ToECDSAPub(common.FromHex(args.From))); key != nil { - options.From = key - } else { - return false, fmt.Errorf("unknown identity to send from: %s", args.From) - } - } - - // Wrap and send the message - pow := time.Duration(args.Priority) * time.Millisecond - envelope, err := message.Wrap(pow, options) - if err != nil { - return false, err - } - - return true, s.w.Send(envelope) -} - -// WhisperMessage is the RPC representation of a whisper message. -type WhisperMessage struct { - ref *Message - - Payload string `json:"payload"` - To string `json:"to"` - From string `json:"from"` - Sent int64 `json:"sent"` - TTL int64 `json:"ttl"` - Hash string `json:"hash"` -} - -func (args *PostArgs) UnmarshalJSON(data []byte) (err error) { - var obj struct { - From string `json:"from"` - To string `json:"to"` - Topics []string `json:"topics"` - Payload string `json:"payload"` - Priority hexutil.Uint64 `json:"priority"` - TTL hexutil.Uint64 `json:"ttl"` - } - - if err := json.Unmarshal(data, &obj); err != nil { - return err - } - - args.From = obj.From - args.To = obj.To - args.Payload = obj.Payload - args.Priority = int64(obj.Priority) // TODO(gluk256): handle overflow - args.TTL = int64(obj.TTL) // ... here too ... - - // decode topic strings - args.Topics = make([][]byte, len(obj.Topics)) - for i, topic := range obj.Topics { - args.Topics[i] = common.FromHex(topic) - } - - return nil -} - -// UnmarshalJSON implements the json.Unmarshaler interface, invoked to convert a -// JSON message blob into a WhisperFilterArgs structure. -func (args *NewFilterArgs) UnmarshalJSON(b []byte) (err error) { - // Unmarshal the JSON message and sanity check - var obj struct { - To interface{} `json:"to"` - From interface{} `json:"from"` - Topics interface{} `json:"topics"` - } - if err := json.Unmarshal(b, &obj); err != nil { - return err - } - - // Retrieve the simple data contents of the filter arguments - if obj.To == nil { - args.To = "" - } else { - argstr, ok := obj.To.(string) - if !ok { - return fmt.Errorf("to is not a string") - } - args.To = argstr - } - if obj.From == nil { - args.From = "" - } else { - argstr, ok := obj.From.(string) - if !ok { - return fmt.Errorf("from is not a string") - } - args.From = argstr - } - // Construct the nested topic array - if obj.Topics != nil { - // Make sure we have an actual topic array - list, ok := obj.Topics.([]interface{}) - if !ok { - return fmt.Errorf("topics is not an array") - } - // Iterate over each topic and handle nil, string or array - topics := make([][]string, len(list)) - for idx, field := range list { - switch value := field.(type) { - case nil: - topics[idx] = []string{} - - case string: - topics[idx] = []string{value} - - case []interface{}: - topics[idx] = make([]string, len(value)) - for i, nested := range value { - switch value := nested.(type) { - case nil: - topics[idx][i] = "" - - case string: - topics[idx][i] = value - - default: - return fmt.Errorf("topic[%d][%d] is not a string", idx, i) - } - } - default: - return fmt.Errorf("topic[%d] not a string or array", idx) - } - } - - topicsDecoded := make([][][]byte, len(topics)) - for i, condition := range topics { - topicsDecoded[i] = make([][]byte, len(condition)) - for j, topic := range condition { - topicsDecoded[i][j] = common.FromHex(topic) - } - } - - args.Topics = topicsDecoded - } - return nil -} - -// whisperFilter is the message cache matching a specific filter, accumulating -// inbound messages until the are requested by the client. -type whisperFilter struct { - id hexutil.Uint // Filter identifier for old message retrieval - ref *Whisper // Whisper reference for old message retrieval - - cache []WhisperMessage // Cache of messages not yet polled - skip map[common.Hash]struct{} // List of retrieved messages to avoid duplication - update time.Time // Time of the last message query - - lock sync.RWMutex // Lock protecting the filter internals -} - -// messages retrieves all the cached messages from the entire pool matching the -// filter, resetting the filter's change buffer. -func (w *whisperFilter) messages() []*Message { - w.lock.Lock() - defer w.lock.Unlock() - - w.cache = nil - w.update = time.Now() - - w.skip = make(map[common.Hash]struct{}) - messages := w.ref.Messages(int(w.id)) - for _, message := range messages { - w.skip[message.Hash] = struct{}{} - } - return messages -} - -// insert injects a new batch of messages into the filter cache. -func (w *whisperFilter) insert(messages ...WhisperMessage) { - w.lock.Lock() - defer w.lock.Unlock() - - for _, message := range messages { - if _, ok := w.skip[message.ref.Hash]; !ok { - w.cache = append(w.cache, messages...) - } - } -} - -// retrieve fetches all the cached messages from the filter. -func (w *whisperFilter) retrieve() (messages []WhisperMessage) { - w.lock.Lock() - defer w.lock.Unlock() - - messages, w.cache = w.cache, nil - w.update = time.Now() - - return -} - -// newWhisperFilter creates a new serialized, poll based whisper topic filter. -func newWhisperFilter(id hexutil.Uint, ref *Whisper) *whisperFilter { - return &whisperFilter{ - id: id, - ref: ref, - update: time.Now(), - skip: make(map[common.Hash]struct{}), - } -} - -// NewWhisperMessage converts an internal message into an API version. -func NewWhisperMessage(message *Message) WhisperMessage { - return WhisperMessage{ - ref: message, - - Payload: common.ToHex(message.Payload), - From: common.ToHex(crypto.FromECDSAPub(message.Recover())), - To: common.ToHex(crypto.FromECDSAPub(message.To)), - Sent: message.Sent.Unix(), - TTL: int64(message.TTL / time.Second), - Hash: common.ToHex(message.Hash.Bytes()), - } -} diff --git a/whisper/whisperv2/doc.go b/whisper/whisperv2/doc.go deleted file mode 100644 index 7252f44b10..0000000000 --- a/whisper/whisperv2/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -/* -Package whisper implements the Whisper PoC-1. - -(https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec) - -Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP). -As such it may be likened and compared to both, not dissimilar to the -matter/energy duality (apologies to physicists for the blatant abuse of a -fundamental and beautiful natural principle). - -Whisper is a pure identity-based messaging system. Whisper provides a low-level -(non-application-specific) but easily-accessible API without being based upon -or prejudiced by the low-level hardware attributes and characteristics, -particularly the notion of singular endpoints. -*/ -package whisperv2 diff --git a/whisper/whisperv2/envelope.go b/whisper/whisperv2/envelope.go deleted file mode 100644 index 9f1c682040..0000000000 --- a/whisper/whisperv2/envelope.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the Whisper protocol Envelope element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#envelopes. - -package whisperv2 - -import ( - "crypto/ecdsa" - "encoding/binary" - "fmt" - "math/big" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/rlp" -) - -// Envelope represents a clear-text data packet to transmit through the Whisper -// network. Its contents may or may not be encrypted and signed. -type Envelope struct { - Expiry uint32 // Whisper protocol specifies int32, really should be int64 - TTL uint32 // ^^^^^^ - Topics []Topic - Data []byte - Nonce uint32 - - hash common.Hash // Cached hash of the envelope to avoid rehashing every time -} - -// NewEnvelope wraps a Whisper message with expiration and destination data -// included into an envelope for network forwarding. -func NewEnvelope(ttl time.Duration, topics []Topic, msg *Message) *Envelope { - return &Envelope{ - Expiry: uint32(time.Now().Add(ttl).Unix()), - TTL: uint32(ttl.Seconds()), - Topics: topics, - Data: msg.bytes(), - Nonce: 0, - } -} - -// Seal closes the envelope by spending the requested amount of time as a proof -// of work on hashing the data. -func (self *Envelope) Seal(pow time.Duration) { - d := make([]byte, 64) - copy(d[:32], self.rlpWithoutNonce()) - - finish, bestBit := time.Now().Add(pow).UnixNano(), 0 - for nonce := uint32(0); time.Now().UnixNano() < finish; { - for i := 0; i < 1024; i++ { - binary.BigEndian.PutUint32(d[60:], nonce) - - d := new(big.Int).SetBytes(crypto.Keccak256(d)) - firstBit := math.FirstBitSet(d) - if firstBit > bestBit { - self.Nonce, bestBit = nonce, firstBit - } - nonce++ - } - } -} - -// rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce. -func (self *Envelope) rlpWithoutNonce() []byte { - enc, _ := rlp.EncodeToBytes([]interface{}{self.Expiry, self.TTL, self.Topics, self.Data}) - return enc -} - -// Open extracts the message contained within a potentially encrypted envelope. -func (self *Envelope) Open(key *ecdsa.PrivateKey) (msg *Message, err error) { - // Split open the payload into a message construct - data := self.Data - - message := &Message{ - Flags: data[0], - Sent: time.Unix(int64(self.Expiry-self.TTL), 0), - TTL: time.Duration(self.TTL) * time.Second, - Hash: self.Hash(), - } - data = data[1:] - - if message.Flags&signatureFlag == signatureFlag { - if len(data) < signatureLength { - return nil, fmt.Errorf("unable to open envelope. First bit set but len(data) < len(signature)") - } - message.Signature, data = data[:signatureLength], data[signatureLength:] - } - message.Payload = data - - // Decrypt the message, if requested - if key == nil { - return message, nil - } - err = message.decrypt(key) - switch err { - case nil: - return message, nil - - case ecies.ErrInvalidPublicKey: // Payload isn't encrypted - return message, err - - default: - return nil, fmt.Errorf("unable to open envelope, decrypt failed: %v", err) - } -} - -// Hash returns the SHA3 hash of the envelope, calculating it if not yet done. -func (self *Envelope) Hash() common.Hash { - if (self.hash == common.Hash{}) { - enc, _ := rlp.EncodeToBytes(self) - self.hash = crypto.Keccak256Hash(enc) - } - return self.hash -} - -// DecodeRLP decodes an Envelope from an RLP data stream. -func (self *Envelope) DecodeRLP(s *rlp.Stream) error { - raw, err := s.Raw() - if err != nil { - return err - } - // The decoding of Envelope uses the struct fields but also needs - // to compute the hash of the whole RLP-encoded envelope. This - // type has the same structure as Envelope but is not an - // rlp.Decoder so we can reuse the Envelope struct definition. - type rlpenv Envelope - if err := rlp.DecodeBytes(raw, (*rlpenv)(self)); err != nil { - return err - } - self.hash = crypto.Keccak256Hash(raw) - return nil -} diff --git a/whisper/whisperv2/envelope_test.go b/whisper/whisperv2/envelope_test.go deleted file mode 100644 index 490ed9f6f8..0000000000 --- a/whisper/whisperv2/envelope_test.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "bytes" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" -) - -func TestEnvelopeOpen(t *testing.T) { - payload := []byte("hello world") - message := NewMessage(payload) - - envelope, err := message.Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.Flags != message.Flags { - t.Fatalf("flags mismatch: have %d, want %d", opened.Flags, message.Flags) - } - if !bytes.Equal(opened.Signature, message.Signature) { - t.Fatalf("signature mismatch: have 0x%x, want 0x%x", opened.Signature, message.Signature) - } - if !bytes.Equal(opened.Payload, message.Payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, message.Payload) - } - if opened.Sent.Unix() != message.Sent.Unix() { - t.Fatalf("send time mismatch: have %v, want %v", opened.Sent, message.Sent) - } - if opened.TTL/time.Second != DefaultTTL/time.Second { - t.Fatalf("message TTL mismatch: have %v, want %v", opened.TTL, DefaultTTL) - } - - if opened.Hash != envelope.Hash() { - t.Fatalf("message hash mismatch: have 0x%x, want 0x%x", opened.Hash, envelope.Hash()) - } -} - -func TestEnvelopeAnonymousOpenUntargeted(t *testing.T) { - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} - -func TestEnvelopeAnonymousOpenTargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload match, should have been encrypted: 0x%x", opened.Payload) - } -} - -func TestEnvelopeIdentifiedOpenUntargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(key) - switch err { - case nil: - t.Fatalf("envelope opened with bad key: %v", opened) - - case ecies.ErrInvalidPublicKey: - // Ok, key mismatch but opened - - default: - t.Fatalf("failed to open envelope: %v", err) - } - - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} - -func TestEnvelopeIdentifiedOpenTargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(key) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} diff --git a/whisper/whisperv2/filter.go b/whisper/whisperv2/filter.go deleted file mode 100644 index 7404859b78..0000000000 --- a/whisper/whisperv2/filter.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the message filter for fine grained subscriptions. - -package whisperv2 - -import ( - "crypto/ecdsa" - - "github.com/ethereum/go-ethereum/event/filter" -) - -// Filter is used to subscribe to specific types of whisper messages. -type Filter struct { - To *ecdsa.PublicKey // Recipient of the message - From *ecdsa.PublicKey // Sender of the message - Topics [][]Topic // Topics to filter messages with - Fn func(msg *Message) // Handler in case of a match -} - -// NewFilterTopics creates a 2D topic array used by whisper.Filter from binary -// data elements. -func NewFilterTopics(data ...[][]byte) [][]Topic { - filter := make([][]Topic, len(data)) - for i, condition := range data { - // Handle the special case of condition == [[]byte{}] - if len(condition) == 1 && len(condition[0]) == 0 { - filter[i] = []Topic{} - continue - } - // Otherwise flatten normally - filter[i] = NewTopics(condition...) - } - return filter -} - -// NewFilterTopicsFlat creates a 2D topic array used by whisper.Filter from flat -// binary data elements. -func NewFilterTopicsFlat(data ...[]byte) [][]Topic { - filter := make([][]Topic, len(data)) - for i, element := range data { - // Only add non-wildcard topics - filter[i] = make([]Topic, 0, 1) - if len(element) > 0 { - filter[i] = append(filter[i], NewTopic(element)) - } - } - return filter -} - -// NewFilterTopicsFromStrings creates a 2D topic array used by whisper.Filter -// from textual data elements. -func NewFilterTopicsFromStrings(data ...[]string) [][]Topic { - filter := make([][]Topic, len(data)) - for i, condition := range data { - // Handle the special case of condition == [""] - if len(condition) == 1 && condition[0] == "" { - filter[i] = []Topic{} - continue - } - // Otherwise flatten normally - filter[i] = NewTopicsFromStrings(condition...) - } - return filter -} - -// NewFilterTopicsFromStringsFlat creates a 2D topic array used by whisper.Filter from flat -// binary data elements. -func NewFilterTopicsFromStringsFlat(data ...string) [][]Topic { - filter := make([][]Topic, len(data)) - for i, element := range data { - // Only add non-wildcard topics - filter[i] = make([]Topic, 0, 1) - if element != "" { - filter[i] = append(filter[i], NewTopicFromString(element)) - } - } - return filter -} - -// filterer is the internal, fully initialized filter ready to match inbound -// messages to a variety of criteria. -type filterer struct { - to string // Recipient of the message - from string // Sender of the message - matcher *topicMatcher // Topics to filter messages with - fn func(data interface{}) // Handler in case of a match -} - -// Compare checks if the specified filter matches the current one. -func (self filterer) Compare(f filter.Filter) bool { - filter := f.(filterer) - - // Check the message sender and recipient - if len(self.to) > 0 && self.to != filter.to { - return false - } - if len(self.from) > 0 && self.from != filter.from { - return false - } - // Check the topic filtering - topics := make([]Topic, len(filter.matcher.conditions)) - for i, group := range filter.matcher.conditions { - // Message should contain a single topic entry, extract - for topics[i] = range group { - break - } - } - return self.matcher.Matches(topics) -} - -// Trigger is called when a filter successfully matches an inbound message. -func (self filterer) Trigger(data interface{}) { - self.fn(data) -} diff --git a/whisper/whisperv2/filter_test.go b/whisper/whisperv2/filter_test.go deleted file mode 100644 index ffdfd7b349..0000000000 --- a/whisper/whisperv2/filter_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "bytes" - - "testing" -) - -var filterTopicsCreationTests = []struct { - topics [][]string - filter [][][4]byte -}{ - { // Simple topic filter - topics: [][]string{ - {"abc", "def", "ghi"}, - {"def"}, - {"ghi", "abc"}, - }, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}, {0x34, 0x60, 0x7c, 0x9b}, {0x21, 0x41, 0x7d, 0xf9}}, - {{0x34, 0x60, 0x7c, 0x9b}}, - {{0x21, 0x41, 0x7d, 0xf9}, {0x4e, 0x03, 0x65, 0x7a}}, - }, - }, - { // Wild-carded topic filter - topics: [][]string{ - {"abc", "def", "ghi"}, - {}, - {""}, - {"def"}, - }, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}, {0x34, 0x60, 0x7c, 0x9b}, {0x21, 0x41, 0x7d, 0xf9}}, - {}, - {}, - {{0x34, 0x60, 0x7c, 0x9b}}, - }, - }, -} - -var filterTopicsCreationFlatTests = []struct { - topics []string - filter [][][4]byte -}{ - { // Simple topic list - topics: []string{"abc", "def", "ghi"}, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}}, - {{0x34, 0x60, 0x7c, 0x9b}}, - {{0x21, 0x41, 0x7d, 0xf9}}, - }, - }, - { // Wild-carded topic list - topics: []string{"abc", "", "ghi"}, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}}, - {}, - {{0x21, 0x41, 0x7d, 0xf9}}, - }, - }, -} - -func TestFilterTopicsCreation(t *testing.T) { - // Check full filter creation - for i, tt := range filterTopicsCreationTests { - // Check the textual creation - filter := NewFilterTopicsFromStrings(tt.topics...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - // Check the binary creation - binary := make([][][]byte, len(tt.topics)) - for j, condition := range tt.topics { - binary[j] = make([][]byte, len(condition)) - for k, segment := range condition { - binary[j][k] = []byte(segment) - } - } - filter = NewFilterTopics(binary...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - } - // Check flat filter creation - for i, tt := range filterTopicsCreationFlatTests { - // Check the textual creation - filter := NewFilterTopicsFromStringsFlat(tt.topics...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - // Check the binary creation - binary := make([][]byte, len(tt.topics)) - for j, topic := range tt.topics { - binary[j] = []byte(topic) - } - filter = NewFilterTopicsFlat(binary...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - } -} - -var filterCompareTests = []struct { - matcher filterer - message filterer - match bool -}{ - { // Wild-card filter matching anything - matcher: filterer{to: "", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter matching the to field - matcher: filterer{to: "to", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the to field - matcher: filterer{to: "to", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: false, - }, - { // Filter matching the from field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the from field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: false, - }, - { // Filter matching the topic field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the topic field - matcher: filterer{to: "", from: "", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher()}, - match: false, - }, -} - -func TestFilterCompare(t *testing.T) { - for i, tt := range filterCompareTests { - if match := tt.matcher.Compare(tt.message); match != tt.match { - t.Errorf("test %d: match mismatch: have %v, want %v", i, match, tt.match) - } - } -} diff --git a/whisper/whisperv2/main.go b/whisper/whisperv2/main.go deleted file mode 100644 index be41604890..0000000000 --- a/whisper/whisperv2/main.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build none - -// Contains a simple whisper peer setup and self messaging to allow playing -// around with the protocol and API without a fancy client implementation. - -package main - -import ( - "fmt" - "log" - "os" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/nat" - "github.com/ethereum/go-ethereum/whisper" -) - -func main() { - logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel)) - - // Generate the peer identity - key, err := crypto.GenerateKey() - if err != nil { - fmt.Printf("Failed to generate peer key: %v.\n", err) - os.Exit(-1) - } - name := common.MakeName("whisper-go", "1.0") - shh := whisper.New() - - // Create an Ethereum peer to communicate through - server := p2p.Server{ - PrivateKey: key, - MaxPeers: 10, - Name: name, - Protocols: []p2p.Protocol{shh.Protocol()}, - ListenAddr: ":30300", - NAT: nat.Any(), - } - fmt.Println("Starting Ethereum peer...") - if err := server.Start(); err != nil { - fmt.Printf("Failed to start Ethereum peer: %v.\n", err) - os.Exit(1) - } - - // Send a message to self to check that something works - payload := fmt.Sprintf("Hello world, this is %v. In case you're wondering, the time is %v", name, time.Now()) - if err := selfSend(shh, []byte(payload)); err != nil { - fmt.Printf("Failed to self message: %v.\n", err) - os.Exit(-1) - } -} - -// SendSelf wraps a payload into a Whisper envelope and forwards it to itself. -func selfSend(shh *whisper.Whisper, payload []byte) error { - ok := make(chan struct{}) - - // Start watching for self messages, output any arrivals - id := shh.NewIdentity() - shh.Watch(whisper.Filter{ - To: &id.PublicKey, - Fn: func(msg *whisper.Message) { - fmt.Printf("Message received: %s, signed with 0x%x.\n", string(msg.Payload), msg.Signature) - close(ok) - }, - }) - // Wrap the payload and encrypt it - msg := whisper.NewMessage(payload) - envelope, err := msg.Wrap(whisper.DefaultPoW, whisper.Options{ - From: id, - To: &id.PublicKey, - TTL: whisper.DefaultTTL, - }) - if err != nil { - return fmt.Errorf("failed to seal message: %v", err) - } - // Dump the message into the system and wait for it to pop back out - if err := shh.Send(envelope); err != nil { - return fmt.Errorf("failed to send self-message: %v", err) - } - select { - case <-ok: - case <-time.After(time.Second): - return fmt.Errorf("failed to receive message in time") - } - return nil -} diff --git a/whisper/whisperv2/message.go b/whisper/whisperv2/message.go deleted file mode 100644 index 66648c3be8..0000000000 --- a/whisper/whisperv2/message.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the Whisper protocol Message element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#messages. - -package whisperv2 - -import ( - "crypto/ecdsa" - crand "crypto/rand" - "fmt" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/log" -) - -// Message represents an end-user data packet to transmit through the Whisper -// protocol. These are wrapped into Envelopes that need not be understood by -// intermediate nodes, just forwarded. -type Message struct { - Flags byte // First bit is signature presence, rest reserved and should be random - Signature []byte - Payload []byte - - Sent time.Time // Time when the message was posted into the network - TTL time.Duration // Maximum time to live allowed for the message - - To *ecdsa.PublicKey // Message recipient (identity used to decode the message) - Hash common.Hash // Message envelope hash to act as a unique id -} - -// Options specifies the exact way a message should be wrapped into an Envelope. -type Options struct { - From *ecdsa.PrivateKey - To *ecdsa.PublicKey - TTL time.Duration - Topics []Topic -} - -// NewMessage creates and initializes a non-signed, non-encrypted Whisper message. -func NewMessage(payload []byte) *Message { - // Construct an initial flag set: no signature, rest random - flags := byte(rand.Intn(256)) - flags &= ^signatureFlag - - // Assemble and return the message - return &Message{ - Flags: flags, - Payload: payload, - Sent: time.Now(), - } -} - -// Wrap bundles the message into an Envelope to transmit over the network. -// -// pow (Proof Of Work) controls how much time to spend on hashing the message, -// inherently controlling its priority through the network (smaller hash, bigger -// priority). -// -// The user can control the amount of identity, privacy and encryption through -// the options parameter as follows: -// - options.From == nil && options.To == nil: anonymous broadcast -// - options.From != nil && options.To == nil: signed broadcast (known sender) -// - options.From == nil && options.To != nil: encrypted anonymous message -// - options.From != nil && options.To != nil: encrypted signed message -func (self *Message) Wrap(pow time.Duration, options Options) (*Envelope, error) { - // Use the default TTL if non was specified - if options.TTL == 0 { - options.TTL = DefaultTTL - } - self.TTL = options.TTL - - // Sign and encrypt the message if requested - if options.From != nil { - if err := self.sign(options.From); err != nil { - return nil, err - } - } - if options.To != nil { - if err := self.encrypt(options.To); err != nil { - return nil, err - } - } - // Wrap the processed message, seal it and return - envelope := NewEnvelope(options.TTL, options.Topics, self) - envelope.Seal(pow) - - return envelope, nil -} - -// sign calculates and sets the cryptographic signature for the message , also -// setting the sign flag. -func (self *Message) sign(key *ecdsa.PrivateKey) (err error) { - self.Flags |= signatureFlag - self.Signature, err = crypto.Sign(self.hash(), key) - return -} - -// Recover retrieves the public key of the message signer. -func (self *Message) Recover() *ecdsa.PublicKey { - defer func() { recover() }() // in case of invalid signature - - // Short circuit if no signature is present - if self.Signature == nil { - return nil - } - // Otherwise try and recover the signature - pub, err := crypto.SigToPub(self.hash(), self.Signature) - if err != nil { - log.Error(fmt.Sprintf("Could not get public key from signature: %v", err)) - return nil - } - return pub -} - -// encrypt encrypts a message payload with a public key. -func (self *Message) encrypt(key *ecdsa.PublicKey) (err error) { - self.Payload, err = ecies.Encrypt(crand.Reader, ecies.ImportECDSAPublic(key), self.Payload, nil, nil) - return -} - -// decrypt decrypts an encrypted payload with a private key. -func (self *Message) decrypt(key *ecdsa.PrivateKey) error { - cleartext, err := ecies.ImportECDSA(key).Decrypt(crand.Reader, self.Payload, nil, nil) - if err == nil { - self.Payload = cleartext - } - return err -} - -// hash calculates the SHA3 checksum of the message flags and payload. -func (self *Message) hash() []byte { - return crypto.Keccak256(append([]byte{self.Flags}, self.Payload...)) -} - -// bytes flattens the message contents (flags, signature and payload) into a -// single binary blob. -func (self *Message) bytes() []byte { - return append([]byte{self.Flags}, append(self.Signature, self.Payload...)...) -} diff --git a/whisper/whisperv2/message_test.go b/whisper/whisperv2/message_test.go deleted file mode 100644 index c760ac54c4..0000000000 --- a/whisper/whisperv2/message_test.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "bytes" - "crypto/elliptic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" -) - -// Tests whether a message can be wrapped without any identity or encryption. -func TestMessageSimpleWrap(t *testing.T) { - payload := []byte("hello world") - - msg := NewMessage(payload) - if _, err := msg.Wrap(DefaultPoW, Options{}); err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if msg.Flags&signatureFlag != 0 { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, 0) - } - if len(msg.Signature) != 0 { - t.Fatalf("signature found for simple wrapping: 0x%x", msg.Signature) - } - if !bytes.Equal(msg.Payload, payload) { - t.Fatalf("payload mismatch after wrapping: have 0x%x, want 0x%x", msg.Payload, payload) - } - if msg.TTL/time.Second != DefaultTTL/time.Second { - t.Fatalf("message TTL mismatch: have %v, want %v", msg.TTL, DefaultTTL) - } -} - -// Tests whether a message can be signed, and wrapped in plain-text. -func TestMessageCleartextSignRecover(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create crypto key: %v", err) - } - payload := []byte("hello world") - - msg := NewMessage(payload) - if _, err := msg.Wrap(DefaultPoW, Options{ - From: key, - }); err != nil { - t.Fatalf("failed to sign message: %v", err) - } - if msg.Flags&signatureFlag != signatureFlag { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, signatureFlag) - } - if !bytes.Equal(msg.Payload, payload) { - t.Fatalf("payload mismatch after signing: have 0x%x, want 0x%x", msg.Payload, payload) - } - - pubKey := msg.Recover() - if pubKey == nil { - t.Fatalf("failed to recover public key") - } - p1 := elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y) - p2 := elliptic.Marshal(crypto.S256(), pubKey.X, pubKey.Y) - if !bytes.Equal(p1, p2) { - t.Fatalf("public key mismatch: have 0x%x, want 0x%x", p2, p1) - } -} - -// Tests whether a message can be encrypted and decrypted using an anonymous -// sender (i.e. no signature). -func TestMessageAnonymousEncryptDecrypt(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create recipient crypto key: %v", err) - } - payload := []byte("hello world") - - msg := NewMessage(payload) - envelope, err := msg.Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to encrypt message: %v", err) - } - if msg.Flags&signatureFlag != 0 { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, 0) - } - if len(msg.Signature) != 0 { - t.Fatalf("signature found for anonymous message: 0x%x", msg.Signature) - } - - out, err := envelope.Open(key) - if err != nil { - t.Fatalf("failed to open encrypted message: %v", err) - } - if !bytes.Equal(out.Payload, payload) { - t.Errorf("payload mismatch: have 0x%x, want 0x%x", out.Payload, payload) - } -} - -// Tests whether a message can be properly signed and encrypted. -func TestMessageFullCrypto(t *testing.T) { - fromKey, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create sender crypto key: %v", err) - } - toKey, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create recipient crypto key: %v", err) - } - - payload := []byte("hello world") - msg := NewMessage(payload) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: fromKey, - To: &toKey.PublicKey, - }) - if err != nil { - t.Fatalf("failed to encrypt message: %v", err) - } - if msg.Flags&signatureFlag != signatureFlag { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, signatureFlag) - } - if len(msg.Signature) == 0 { - t.Fatalf("no signature found for signed message") - } - - out, err := envelope.Open(toKey) - if err != nil { - t.Fatalf("failed to open encrypted message: %v", err) - } - if !bytes.Equal(out.Payload, payload) { - t.Errorf("payload mismatch: have 0x%x, want 0x%x", out.Payload, payload) - } - - pubKey := out.Recover() - if pubKey == nil { - t.Fatalf("failed to recover public key") - } - p1 := elliptic.Marshal(crypto.S256(), fromKey.PublicKey.X, fromKey.PublicKey.Y) - p2 := elliptic.Marshal(crypto.S256(), pubKey.X, pubKey.Y) - if !bytes.Equal(p1, p2) { - t.Fatalf("public key mismatch: have 0x%x, want 0x%x", p2, p1) - } -} diff --git a/whisper/whisperv2/peer.go b/whisper/whisperv2/peer.go deleted file mode 100644 index 71798408b9..0000000000 --- a/whisper/whisperv2/peer.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "fmt" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/fatih/set.v0" -) - -// peer represents a whisper protocol peer connection. -type peer struct { - host *Whisper - peer *p2p.Peer - ws p2p.MsgReadWriter - - known *set.Set // Messages already known by the peer to avoid wasting bandwidth - - quit chan struct{} -} - -// newPeer creates a new whisper peer object, but does not run the handshake itself. -func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *peer { - return &peer{ - host: host, - peer: remote, - ws: rw, - known: set.New(), - quit: make(chan struct{}), - } -} - -// start initiates the peer updater, periodically broadcasting the whisper packets -// into the network. -func (self *peer) start() { - go self.update() - log.Debug(fmt.Sprintf("%v: whisper started", self.peer)) -} - -// stop terminates the peer updater, stopping message forwarding to it. -func (self *peer) stop() { - close(self.quit) - log.Debug(fmt.Sprintf("%v: whisper stopped", self.peer)) -} - -// handshake sends the protocol initiation status message to the remote peer and -// verifies the remote status too. -func (self *peer) handshake() error { - // Send the handshake status message asynchronously - errc := make(chan error, 1) - go func() { - errc <- p2p.SendItems(self.ws, statusCode, protocolVersion) - }() - // Fetch the remote status packet and verify protocol match - packet, err := self.ws.ReadMsg() - if err != nil { - return err - } - if packet.Code != statusCode { - return fmt.Errorf("peer sent %x before status packet", packet.Code) - } - s := rlp.NewStream(packet.Payload, uint64(packet.Size)) - if _, err := s.List(); err != nil { - return fmt.Errorf("bad status message: %v", err) - } - peerVersion, err := s.Uint() - if err != nil { - return fmt.Errorf("bad status message: %v", err) - } - if peerVersion != protocolVersion { - return fmt.Errorf("protocol version mismatch %d != %d", peerVersion, protocolVersion) - } - // Wait until out own status is consumed too - if err := <-errc; err != nil { - return fmt.Errorf("failed to send status packet: %v", err) - } - return nil -} - -// update executes periodic operations on the peer, including message transmission -// and expiration. -func (self *peer) update() { - // Start the tickers for the updates - expire := time.NewTicker(expirationCycle) - transmit := time.NewTicker(transmissionCycle) - - // Loop and transmit until termination is requested - for { - select { - case <-expire.C: - self.expire() - - case <-transmit.C: - if err := self.broadcast(); err != nil { - log.Info(fmt.Sprintf("%v: broadcast failed: %v", self.peer, err)) - return - } - - case <-self.quit: - return - } - } -} - -// mark marks an envelope known to the peer so that it won't be sent back. -func (self *peer) mark(envelope *Envelope) { - self.known.Add(envelope.Hash()) -} - -// marked checks if an envelope is already known to the remote peer. -func (self *peer) marked(envelope *Envelope) bool { - return self.known.Has(envelope.Hash()) -} - -// expire iterates over all the known envelopes in the host and removes all -// expired (unknown) ones from the known list. -func (self *peer) expire() { - // Assemble the list of available envelopes - available := set.NewNonTS() - for _, envelope := range self.host.envelopes() { - available.Add(envelope.Hash()) - } - // Cross reference availability with known status - unmark := make(map[common.Hash]struct{}) - self.known.Each(func(v interface{}) bool { - if !available.Has(v.(common.Hash)) { - unmark[v.(common.Hash)] = struct{}{} - } - return true - }) - // Dump all known but unavailable - for hash := range unmark { - self.known.Remove(hash) - } -} - -// broadcast iterates over the collection of envelopes and transmits yet unknown -// ones over the network. -func (self *peer) broadcast() error { - // Fetch the envelopes and collect the unknown ones - envelopes := self.host.envelopes() - transmit := make([]*Envelope, 0, len(envelopes)) - for _, envelope := range envelopes { - if !self.marked(envelope) { - transmit = append(transmit, envelope) - self.mark(envelope) - } - } - // Transmit the unknown batch (potentially empty) - if err := p2p.Send(self.ws, messagesCode, transmit); err != nil { - return err - } - log.Trace(fmt.Sprint(self.peer, "broadcasted", len(transmit), "message(s)")) - return nil -} diff --git a/whisper/whisperv2/peer_test.go b/whisper/whisperv2/peer_test.go deleted file mode 100644 index 87ca5063df..0000000000 --- a/whisper/whisperv2/peer_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "testing" - "time" - - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" -) - -type testPeer struct { - client *Whisper - stream *p2p.MsgPipeRW - termed chan struct{} -} - -func startTestPeer() *testPeer { - // Create a simulated P2P remote peer and data streams to it - remote := p2p.NewPeer(discover.NodeID{}, "", nil) - tester, tested := p2p.MsgPipe() - - // Create a whisper client and connect with it to the tester peer - client := New() - client.Start(nil) - - termed := make(chan struct{}) - go func() { - defer client.Stop() - defer close(termed) - defer tested.Close() - - client.handlePeer(remote, tested) - }() - - return &testPeer{ - client: client, - stream: tester, - termed: termed, - } -} - -func startTestPeerInited() (*testPeer, error) { - peer := startTestPeer() - - if err := p2p.ExpectMsg(peer.stream, statusCode, []uint64{protocolVersion}); err != nil { - peer.stream.Close() - return nil, err - } - if err := p2p.SendItems(peer.stream, statusCode, protocolVersion); err != nil { - peer.stream.Close() - return nil, err - } - return peer, nil -} - -func TestPeerStatusMessage(t *testing.T) { - tester := startTestPeer() - - // Wait for the handshake status message and check it - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Terminate the node - tester.stream.Close() - - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("local close timed out") - } -} - -func TestPeerHandshakeFail(t *testing.T) { - tester := startTestPeer() - - // Wait for and check the handshake - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Send an invalid handshake status and verify disconnect - if err := p2p.SendItems(tester.stream, messagesCode); err != nil { - t.Fatalf("failed to send malformed status: %v", err) - } - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("remote close timed out") - } -} - -func TestPeerHandshakeSuccess(t *testing.T) { - tester := startTestPeer() - - // Wait for and check the handshake - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Send a valid handshake status and make sure connection stays live - if err := p2p.SendItems(tester.stream, statusCode, protocolVersion); err != nil { - t.Fatalf("failed to send status: %v", err) - } - select { - case <-tester.termed: - t.Fatalf("valid handshake disconnected") - - case <-time.After(100 * time.Millisecond): - } - // Clean up the test - tester.stream.Close() - - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("local close timed out") - } -} - -func TestPeerSend(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Construct a message and inject into the tester - message := NewMessage([]byte("peer broadcast test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - // Check that the message is eventually forwarded - payload := []interface{}{envelope} - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - t.Fatalf("message mismatch: %v", err) - } - // Make sure that even with a re-insert, an empty batch is received - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - if err := p2p.ExpectMsg(tester.stream, messagesCode, []interface{}{}); err != nil { - t.Fatalf("message mismatch: %v", err) - } -} - -func TestPeerDeliver(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Watch for all inbound messages - arrived := make(chan struct{}, 1) - tester.client.Watch(Filter{ - Fn: func(message *Message) { - arrived <- struct{}{} - }, - }) - // Construct a message and deliver it to the tester peer - message := NewMessage([]byte("peer broadcast test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := p2p.Send(tester.stream, messagesCode, []*Envelope{envelope}); err != nil { - t.Fatalf("failed to transfer message: %v", err) - } - // Check that the message is delivered upstream - select { - case <-arrived: - case <-time.After(time.Second): - t.Fatalf("message delivery timeout") - } - // Check that a resend is not delivered - if err := p2p.Send(tester.stream, messagesCode, []*Envelope{envelope}); err != nil { - t.Fatalf("failed to transfer message: %v", err) - } - select { - case <-time.After(2 * transmissionCycle): - case <-arrived: - t.Fatalf("repeating message arrived") - } -} - -func TestPeerMessageExpiration(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Fetch the peer instance for later inspection - tester.client.peerMu.RLock() - if peers := len(tester.client.peers); peers != 1 { - t.Fatalf("peer pool size mismatch: have %v, want %v", peers, 1) - } - var peer *peer - for peer = range tester.client.peers { - break - } - tester.client.peerMu.RUnlock() - - // Construct a message and pass it through the tester - message := NewMessage([]byte("peer test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: time.Second, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - payload := []interface{}{envelope} - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - // A premature empty message may have been broadcast, check the next too - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - t.Fatalf("message mismatch: %v", err) - } - } - // Check that the message is inside the cache - if !peer.known.Has(envelope.Hash()) { - t.Fatalf("message not found in cache") - } - // Discard messages until expiration and check cache again - exp := time.Now().Add(time.Second + 2*expirationCycle + 100*time.Millisecond) - for time.Now().Before(exp) { - if err := p2p.ExpectMsg(tester.stream, messagesCode, []interface{}{}); err != nil { - t.Fatalf("message mismatch: %v", err) - } - } - if peer.known.Has(envelope.Hash()) { - t.Fatalf("message not expired from cache") - } -} diff --git a/whisper/whisperv2/topic.go b/whisper/whisperv2/topic.go deleted file mode 100644 index 3e2b47bd3f..0000000000 --- a/whisper/whisperv2/topic.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the Whisper protocol Topic element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#topics. - -package whisperv2 - -import "github.com/ethereum/go-ethereum/crypto" - -// Topic represents a cryptographically secure, probabilistic partial -// classifications of a message, determined as the first (left) 4 bytes of the -// SHA3 hash of some arbitrary data given by the original author of the message. -type Topic [4]byte - -// NewTopic creates a topic from the 4 byte prefix of the SHA3 hash of the data. -// -// Note, empty topics are considered the wildcard, and cannot be used in messages. -func NewTopic(data []byte) Topic { - prefix := [4]byte{} - copy(prefix[:], crypto.Keccak256(data)[:4]) - return Topic(prefix) -} - -// NewTopics creates a list of topics from a list of binary data elements, by -// iteratively calling NewTopic on each of them. -func NewTopics(data ...[]byte) []Topic { - topics := make([]Topic, len(data)) - for i, element := range data { - topics[i] = NewTopic(element) - } - return topics -} - -// NewTopicFromString creates a topic using the binary data contents of the -// specified string. -func NewTopicFromString(data string) Topic { - return NewTopic([]byte(data)) -} - -// NewTopicsFromStrings creates a list of topics from a list of textual data -// elements, by iteratively calling NewTopicFromString on each of them. -func NewTopicsFromStrings(data ...string) []Topic { - topics := make([]Topic, len(data)) - for i, element := range data { - topics[i] = NewTopicFromString(element) - } - return topics -} - -// String converts a topic byte array to a string representation. -func (self *Topic) String() string { - return string(self[:]) -} - -// topicMatcher is a filter expression to verify if a list of topics contained -// in an arriving message matches some topic conditions. The topic matcher is -// built up of a list of conditions, each of which must be satisfied by the -// corresponding topic in the message. Each condition may require: a) an exact -// topic match; b) a match from a set of topics; or c) a wild-card matching all. -// -// If a message contains more topics than required by the matcher, those beyond -// the condition count are ignored and assumed to match. -// -// Consider the following sample topic matcher: -// sample := { -// {TopicA1, TopicA2, TopicA3}, -// {TopicB}, -// nil, -// {TopicD1, TopicD2} -// } -// In order for a message to pass this filter, it should enumerate at least 4 -// topics, the first any of [TopicA1, TopicA2, TopicA3], the second mandatory -// "TopicB", the third is ignored by the filter and the fourth either "TopicD1" -// or "TopicD2". If the message contains further topics, the filter will match -// them too. -type topicMatcher struct { - conditions []map[Topic]struct{} -} - -// newTopicMatcher create a topic matcher from a list of topic conditions. -func newTopicMatcher(topics ...[]Topic) *topicMatcher { - matcher := make([]map[Topic]struct{}, len(topics)) - for i, condition := range topics { - matcher[i] = make(map[Topic]struct{}) - for _, topic := range condition { - matcher[i][topic] = struct{}{} - } - } - return &topicMatcher{conditions: matcher} -} - -// newTopicMatcherFromBinary create a topic matcher from a list of binary conditions. -func newTopicMatcherFromBinary(data ...[][]byte) *topicMatcher { - topics := make([][]Topic, len(data)) - for i, condition := range data { - topics[i] = NewTopics(condition...) - } - return newTopicMatcher(topics...) -} - -// newTopicMatcherFromStrings creates a topic matcher from a list of textual -// conditions. -func newTopicMatcherFromStrings(data ...[]string) *topicMatcher { - topics := make([][]Topic, len(data)) - for i, condition := range data { - topics[i] = NewTopicsFromStrings(condition...) - } - return newTopicMatcher(topics...) -} - -// Matches checks if a list of topics matches this particular condition set. -func (self *topicMatcher) Matches(topics []Topic) bool { - // Mismatch if there aren't enough topics - if len(self.conditions) > len(topics) { - return false - } - // Check each topic condition for existence (skip wild-cards) - for i := 0; i < len(topics) && i < len(self.conditions); i++ { - if len(self.conditions[i]) > 0 { - if _, ok := self.conditions[i][topics[i]]; !ok { - return false - } - } - } - return true -} diff --git a/whisper/whisperv2/topic_test.go b/whisper/whisperv2/topic_test.go deleted file mode 100644 index bb65689963..0000000000 --- a/whisper/whisperv2/topic_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "bytes" - "testing" -) - -var topicCreationTests = []struct { - data []byte - hash [4]byte -}{ - {hash: [4]byte{0x8f, 0x9a, 0x2b, 0x7d}, data: []byte("test name")}, - {hash: [4]byte{0xf2, 0x6e, 0x77, 0x79}, data: []byte("some other test")}, -} - -func TestTopicCreation(t *testing.T) { - // Create the topics individually - for i, tt := range topicCreationTests { - topic := NewTopic(tt.data) - if !bytes.Equal(topic[:], tt.hash[:]) { - t.Errorf("binary test %d: hash mismatch: have %v, want %v.", i, topic, tt.hash) - } - } - for i, tt := range topicCreationTests { - topic := NewTopicFromString(string(tt.data)) - if !bytes.Equal(topic[:], tt.hash[:]) { - t.Errorf("textual test %d: hash mismatch: have %v, want %v.", i, topic, tt.hash) - } - } - // Create the topics in batches - binaryData := make([][]byte, len(topicCreationTests)) - for i, tt := range topicCreationTests { - binaryData[i] = tt.data - } - textualData := make([]string, len(topicCreationTests)) - for i, tt := range topicCreationTests { - textualData[i] = string(tt.data) - } - - topics := NewTopics(binaryData...) - for i, tt := range topicCreationTests { - if !bytes.Equal(topics[i][:], tt.hash[:]) { - t.Errorf("binary batch test %d: hash mismatch: have %v, want %v.", i, topics[i], tt.hash) - } - } - topics = NewTopicsFromStrings(textualData...) - for i, tt := range topicCreationTests { - if !bytes.Equal(topics[i][:], tt.hash[:]) { - t.Errorf("textual batch test %d: hash mismatch: have %v, want %v.", i, topics[i], tt.hash) - } - } -} - -var topicMatcherCreationTest = struct { - binary [][][]byte - textual [][]string - matcher []map[[4]byte]struct{} -}{ - binary: [][][]byte{ - {}, - { - []byte("Topic A"), - }, - { - []byte("Topic B1"), - []byte("Topic B2"), - []byte("Topic B3"), - }, - }, - textual: [][]string{ - {}, - {"Topic A"}, - {"Topic B1", "Topic B2", "Topic B3"}, - }, - matcher: []map[[4]byte]struct{}{ - {}, - { - {0x25, 0xfc, 0x95, 0x66}: {}, - }, - { - {0x93, 0x6d, 0xec, 0x09}: {}, - {0x25, 0x23, 0x34, 0xd3}: {}, - {0x6b, 0xc2, 0x73, 0xd1}: {}, - }, - }, -} - -func TestTopicMatcherCreation(t *testing.T) { - test := topicMatcherCreationTest - - matcher := newTopicMatcherFromBinary(test.binary...) - for i, cond := range matcher.conditions { - for topic := range cond { - if _, ok := test.matcher[i][topic]; !ok { - t.Errorf("condition %d; extra topic found: 0x%x", i, topic[:]) - } - } - } - for i, cond := range test.matcher { - for topic := range cond { - if _, ok := matcher.conditions[i][topic]; !ok { - t.Errorf("condition %d; topic not found: 0x%x", i, topic[:]) - } - } - } - - matcher = newTopicMatcherFromStrings(test.textual...) - for i, cond := range matcher.conditions { - for topic := range cond { - if _, ok := test.matcher[i][topic]; !ok { - t.Errorf("condition %d; extra topic found: 0x%x", i, topic[:]) - } - } - } - for i, cond := range test.matcher { - for topic := range cond { - if _, ok := matcher.conditions[i][topic]; !ok { - t.Errorf("condition %d; topic not found: 0x%x", i, topic[:]) - } - } - } -} - -var topicMatcherTests = []struct { - filter [][]string - topics []string - match bool -}{ - // Empty topic matcher should match everything - { - filter: [][]string{}, - topics: []string{}, - match: true, - }, - { - filter: [][]string{}, - topics: []string{"a", "b", "c"}, - match: true, - }, - // Fixed topic matcher should match strictly, but only prefix - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a", "b"}, - match: true, - }, - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a", "b", "c"}, - match: true, - }, - // Multi-matcher should match any from a sub-group - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a1"}, - match: true, - }, - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a2"}, - match: true, - }, - // Wild-card condition should match anything - { - filter: [][]string{{}, {"b"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{}, {"b"}}, - topics: []string{"a", "b"}, - match: true, - }, - { - filter: [][]string{{}, {"b"}}, - topics: []string{"b", "b"}, - match: true, - }, -} - -func TestTopicMatcher(t *testing.T) { - for i, tt := range topicMatcherTests { - topics := NewTopicsFromStrings(tt.topics...) - - matcher := newTopicMatcherFromStrings(tt.filter...) - if match := matcher.Matches(topics); match != tt.match { - t.Errorf("test %d: match mismatch: have %v, want %v", i, match, tt.match) - } - } -} diff --git a/whisper/whisperv2/whisper.go b/whisper/whisperv2/whisper.go deleted file mode 100644 index e111a34146..0000000000 --- a/whisper/whisperv2/whisper.go +++ /dev/null @@ -1,378 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "crypto/ecdsa" - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/event/filter" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - - "gopkg.in/fatih/set.v0" -) - -const ( - statusCode = 0x00 - messagesCode = 0x01 - - protocolVersion uint64 = 0x02 - protocolName = "shh" - - signatureFlag = byte(1 << 7) - signatureLength = 65 - - expirationCycle = 800 * time.Millisecond - transmissionCycle = 300 * time.Millisecond -) - -const ( - DefaultTTL = 50 * time.Second - DefaultPoW = 50 * time.Millisecond -) - -type MessageEvent struct { - To *ecdsa.PrivateKey - From *ecdsa.PublicKey - Message *Message -} - -// Whisper represents a dark communication interface through the Ethereum -// network, using its very own P2P communication layer. -type Whisper struct { - protocol p2p.Protocol - filters *filter.Filters - - keys map[string]*ecdsa.PrivateKey - - messages map[common.Hash]*Envelope // Pool of messages currently tracked by this node - expirations map[uint32]*set.SetNonTS // Message expiration pool (TODO: something lighter) - poolMu sync.RWMutex // Mutex to sync the message and expiration pools - - peers map[*peer]struct{} // Set of currently active peers - peerMu sync.RWMutex // Mutex to sync the active peer set - - quit chan struct{} -} - -// New creates a Whisper client ready to communicate through the Ethereum P2P -// network. -func New() *Whisper { - whisper := &Whisper{ - filters: filter.New(), - keys: make(map[string]*ecdsa.PrivateKey), - messages: make(map[common.Hash]*Envelope), - expirations: make(map[uint32]*set.SetNonTS), - peers: make(map[*peer]struct{}), - quit: make(chan struct{}), - } - whisper.filters.Start() - - // p2p whisper sub protocol handler - whisper.protocol = p2p.Protocol{ - Name: protocolName, - Version: uint(protocolVersion), - Length: 2, - Run: whisper.handlePeer, - } - - return whisper -} - -// APIs returns the RPC descriptors the Whisper implementation offers -func (s *Whisper) APIs() []rpc.API { - return []rpc.API{ - { - Namespace: "shh", - Version: "1.0", - Service: NewPublicWhisperAPI(s), - Public: true, - }, - } -} - -// Protocols returns the whisper sub-protocols ran by this particular client. -func (self *Whisper) Protocols() []p2p.Protocol { - return []p2p.Protocol{self.protocol} -} - -// Version returns the whisper sub-protocols version number. -func (self *Whisper) Version() uint { - return self.protocol.Version -} - -// NewIdentity generates a new cryptographic identity for the client, and injects -// it into the known identities for message decryption. -func (self *Whisper) NewIdentity() *ecdsa.PrivateKey { - key, err := crypto.GenerateKey() - if err != nil { - panic(err) - } - self.keys[string(crypto.FromECDSAPub(&key.PublicKey))] = key - - return key -} - -// HasIdentity checks if the the whisper node is configured with the private key -// of the specified public pair. -func (self *Whisper) HasIdentity(key *ecdsa.PublicKey) bool { - return self.keys[string(crypto.FromECDSAPub(key))] != nil -} - -// GetIdentity retrieves the private key of the specified public identity. -func (self *Whisper) GetIdentity(key *ecdsa.PublicKey) *ecdsa.PrivateKey { - return self.keys[string(crypto.FromECDSAPub(key))] -} - -// Watch installs a new message handler to run in case a matching packet arrives -// from the whisper network. -func (self *Whisper) Watch(options Filter) int { - filter := filterer{ - to: string(crypto.FromECDSAPub(options.To)), - from: string(crypto.FromECDSAPub(options.From)), - matcher: newTopicMatcher(options.Topics...), - fn: func(data interface{}) { - options.Fn(data.(*Message)) - }, - } - return self.filters.Install(filter) -} - -// Unwatch removes an installed message handler. -func (self *Whisper) Unwatch(id int) { - self.filters.Uninstall(id) -} - -// Send injects a message into the whisper send queue, to be distributed in the -// network in the coming cycles. -func (self *Whisper) Send(envelope *Envelope) error { - return self.add(envelope) -} - -// Start implements node.Service, starting the background data propagation thread -// of the Whisper protocol. -func (self *Whisper) Start(*p2p.Server) error { - log.Info("Whisper started") - go self.update() - return nil -} - -// Stop implements node.Service, stopping the background data propagation thread -// of the Whisper protocol. -func (self *Whisper) Stop() error { - close(self.quit) - log.Info("Whisper stopped") - return nil -} - -// Messages retrieves all the currently pooled messages matching a filter id. -func (self *Whisper) Messages(id int) []*Message { - messages := make([]*Message, 0) - if filter := self.filters.Get(id); filter != nil { - for _, envelope := range self.messages { - if message := self.open(envelope); message != nil { - if self.filters.Match(filter, createFilter(message, envelope.Topics)) { - messages = append(messages, message) - } - } - } - } - return messages -} - -// handlePeer is called by the underlying P2P layer when the whisper sub-protocol -// connection is negotiated. -func (self *Whisper) handlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { - // Create the new peer and start tracking it - whisperPeer := newPeer(self, peer, rw) - - self.peerMu.Lock() - self.peers[whisperPeer] = struct{}{} - self.peerMu.Unlock() - - defer func() { - self.peerMu.Lock() - delete(self.peers, whisperPeer) - self.peerMu.Unlock() - }() - - // Run the peer handshake and state updates - if err := whisperPeer.handshake(); err != nil { - return err - } - whisperPeer.start() - defer whisperPeer.stop() - - // Read and process inbound messages directly to merge into client-global state - for { - // Fetch the next packet and decode the contained envelopes - packet, err := rw.ReadMsg() - if err != nil { - return err - } - var envelopes []*Envelope - if err := packet.Decode(&envelopes); err != nil { - log.Info(fmt.Sprintf("%v: failed to decode envelope: %v", peer, err)) - continue - } - // Inject all envelopes into the internal pool - for _, envelope := range envelopes { - if err := self.add(envelope); err != nil { - // TODO Punish peer here. Invalid envelope. - log.Debug(fmt.Sprintf("%v: failed to pool envelope: %v", peer, err)) - } - whisperPeer.mark(envelope) - } - } -} - -// add inserts a new envelope into the message pool to be distributed within the -// whisper network. It also inserts the envelope into the expiration pool at the -// appropriate time-stamp. -func (self *Whisper) add(envelope *Envelope) error { - self.poolMu.Lock() - defer self.poolMu.Unlock() - - // short circuit when a received envelope has already expired - if envelope.Expiry < uint32(time.Now().Unix()) { - return nil - } - - // Insert the message into the tracked pool - hash := envelope.Hash() - if _, ok := self.messages[hash]; ok { - log.Trace(fmt.Sprintf("whisper envelope already cached: %x\n", hash)) - return nil - } - self.messages[hash] = envelope - - // Insert the message into the expiration pool for later removal - if self.expirations[envelope.Expiry] == nil { - self.expirations[envelope.Expiry] = set.NewNonTS() - } - if !self.expirations[envelope.Expiry].Has(hash) { - self.expirations[envelope.Expiry].Add(hash) - - // Notify the local node of a message arrival - go self.postEvent(envelope) - } - log.Trace(fmt.Sprintf("cached whisper envelope %x\n", hash)) - return nil -} - -// postEvent opens an envelope with the configured identities and delivers the -// message upstream from application processing. -func (self *Whisper) postEvent(envelope *Envelope) { - if message := self.open(envelope); message != nil { - self.filters.Notify(createFilter(message, envelope.Topics), message) - } -} - -// open tries to decrypt a whisper envelope with all the configured identities, -// returning the decrypted message and the key used to achieve it. If not keys -// are configured, open will return the payload as if non encrypted. -func (self *Whisper) open(envelope *Envelope) *Message { - // Short circuit if no identity is set, and assume clear-text - if len(self.keys) == 0 { - if message, err := envelope.Open(nil); err == nil { - return message - } - } - // Iterate over the keys and try to decrypt the message - for _, key := range self.keys { - message, err := envelope.Open(key) - if err == nil { - message.To = &key.PublicKey - return message - } else if err == ecies.ErrInvalidPublicKey { - return message - } - } - // Failed to decrypt, don't return anything - return nil -} - -// createFilter creates a message filter to check against installed handlers. -func createFilter(message *Message, topics []Topic) filter.Filter { - matcher := make([][]Topic, len(topics)) - for i, topic := range topics { - matcher[i] = []Topic{topic} - } - return filterer{ - to: string(crypto.FromECDSAPub(message.To)), - from: string(crypto.FromECDSAPub(message.Recover())), - matcher: newTopicMatcher(matcher...), - } -} - -// update loops until the lifetime of the whisper node, updating its internal -// state by expiring stale messages from the pool. -func (self *Whisper) update() { - // Start a ticker to check for expirations - expire := time.NewTicker(expirationCycle) - - // Repeat updates until termination is requested - for { - select { - case <-expire.C: - self.expire() - - case <-self.quit: - return - } - } -} - -// expire iterates over all the expiration timestamps, removing all stale -// messages from the pools. -func (self *Whisper) expire() { - self.poolMu.Lock() - defer self.poolMu.Unlock() - - now := uint32(time.Now().Unix()) - for then, hashSet := range self.expirations { - // Short circuit if a future time - if then > now { - continue - } - // Dump all expired messages and remove timestamp - hashSet.Each(func(v interface{}) bool { - delete(self.messages, v.(common.Hash)) - return true - }) - self.expirations[then].Clear() - } -} - -// envelopes retrieves all the messages currently pooled by the node. -func (self *Whisper) envelopes() []*Envelope { - self.poolMu.RLock() - defer self.poolMu.RUnlock() - - envelopes := make([]*Envelope, 0, len(self.messages)) - for _, envelope := range self.messages { - envelopes = append(envelopes, envelope) - } - return envelopes -} diff --git a/whisper/whisperv2/whisper_test.go b/whisper/whisperv2/whisper_test.go deleted file mode 100644 index 1e0d3f85d4..0000000000 --- a/whisper/whisperv2/whisper_test.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv2 - -import ( - "testing" - "time" - - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" -) - -func startTestCluster(n int) []*Whisper { - // Create the batch of simulated peers - nodes := make([]*p2p.Peer, n) - for i := 0; i < n; i++ { - nodes[i] = p2p.NewPeer(discover.NodeID{}, "", nil) - } - whispers := make([]*Whisper, n) - for i := 0; i < n; i++ { - whispers[i] = New() - whispers[i].Start(nil) - } - // Wire all the peers to the root one - for i := 1; i < n; i++ { - src, dst := p2p.MsgPipe() - - go whispers[0].handlePeer(nodes[i], src) - go whispers[i].handlePeer(nodes[0], dst) - } - return whispers -} - -func TestSelfMessage(t *testing.T) { - // Start the single node cluster - client := startTestCluster(1)[0] - - // Start watching for self messages, signal any arrivals - self := client.NewIdentity() - done := make(chan struct{}) - - client.Watch(Filter{ - To: &self.PublicKey, - Fn: func(msg *Message) { - close(done) - }, - }) - // Send a dummy message to oneself - msg := NewMessage([]byte("self whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: self, - To: &self.PublicKey, - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - // Dump the message into the system and wait for it to pop back out - if err := client.Send(envelope); err != nil { - t.Fatalf("failed to send self-message: %v", err) - } - select { - case <-done: - case <-time.After(time.Second): - t.Fatalf("self-message receive timeout") - } -} - -func TestDirectMessage(t *testing.T) { - // Start the sender-recipient cluster - cluster := startTestCluster(2) - - sender := cluster[0] - senderId := sender.NewIdentity() - - recipient := cluster[1] - recipientId := recipient.NewIdentity() - - // Watch for arriving messages on the recipient - done := make(chan struct{}) - recipient.Watch(Filter{ - To: &recipientId.PublicKey, - Fn: func(msg *Message) { - close(done) - }, - }) - // Send a dummy message from the sender - msg := NewMessage([]byte("direct whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: senderId, - To: &recipientId.PublicKey, - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := sender.Send(envelope); err != nil { - t.Fatalf("failed to send direct message: %v", err) - } - // Wait for an arrival or a timeout - select { - case <-done: - case <-time.After(time.Second): - t.Fatalf("direct message receive timeout") - } -} - -func TestAnonymousBroadcast(t *testing.T) { - testBroadcast(true, t) -} - -func TestIdentifiedBroadcast(t *testing.T) { - testBroadcast(false, t) -} - -func testBroadcast(anonymous bool, t *testing.T) { - // Start the single sender multi recipient cluster - cluster := startTestCluster(3) - - sender := cluster[1] - targets := cluster[1:] - for _, target := range targets { - if !anonymous { - target.NewIdentity() - } - } - // Watch for arriving messages on the recipients - dones := make([]chan struct{}, len(targets)) - for i := 0; i < len(targets); i++ { - done := make(chan struct{}) // need for the closure - dones[i] = done - - targets[i].Watch(Filter{ - Topics: NewFilterTopicsFromStringsFlat("broadcast topic"), - Fn: func(msg *Message) { - close(done) - }, - }) - } - // Send a dummy message from the sender - msg := NewMessage([]byte("broadcast whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - Topics: NewTopicsFromStrings("broadcast topic"), - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := sender.Send(envelope); err != nil { - t.Fatalf("failed to send broadcast message: %v", err) - } - // Wait for an arrival on each recipient, or timeouts - timeout := time.After(time.Second) - for _, done := range dones { - select { - case <-done: - case <-timeout: - t.Fatalf("broadcast message receive timeout") - } - } -} - -func TestMessageExpiration(t *testing.T) { - // Start the single node cluster and inject a dummy message - node := startTestCluster(1)[0] - - message := NewMessage([]byte("expiring message")) - envelope, err := message.Wrap(DefaultPoW, Options{TTL: time.Second}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := node.Send(envelope); err != nil { - t.Fatalf("failed to inject message: %v", err) - } - // Check that the message is inside the cache - node.poolMu.RLock() - _, found := node.messages[envelope.Hash()] - node.poolMu.RUnlock() - - if !found { - t.Fatalf("message not found in cache") - } - // Wait for expiration and check cache again - time.Sleep(time.Second) // wait for expiration - time.Sleep(2 * expirationCycle) // wait for cleanup cycle - - node.poolMu.RLock() - _, found = node.messages[envelope.Hash()] - node.poolMu.RUnlock() - if found { - t.Fatalf("message not expired from cache") - } - - // Check that adding an expired envelope doesn't do anything. - node.add(envelope) - node.poolMu.RLock() - _, found = node.messages[envelope.Hash()] - node.poolMu.RUnlock() - if found { - t.Fatalf("message was added to cache") - } -} diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index b4494d0d61..ee566625c9 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -60,32 +60,9 @@ func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { w: w, lastUsed: make(map[string]time.Time), } - - go api.run() return api } -// run the api event loop. -// this loop deletes filter that have not been used within filterTimeout -func (api *PublicWhisperAPI) run() { - timeout := time.NewTicker(2 * time.Minute) - for { - <-timeout.C - - api.mu.Lock() - for id, lastUsed := range api.lastUsed { - if time.Since(lastUsed).Seconds() >= filterTimeout { - delete(api.lastUsed, id) - if err := api.w.Unsubscribe(id); err != nil { - log.Error("could not unsubscribe whisper filter", "error", err) - } - log.Debug("delete whisper filter (timeout)", "id", id) - } - } - api.mu.Unlock() - } -} - // Version returns the Whisper sub-protocol version. func (api *PublicWhisperAPI) Version(ctx context.Context) string { return ProtocolVersionStr diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index 8ae2882e1f..96e2b17e7c 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -61,32 +61,9 @@ func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { w: w, lastUsed: make(map[string]time.Time), } - - go api.run() return api } -// run the api event loop. -// this loop deletes filter that have not been used within filterTimeout -func (api *PublicWhisperAPI) run() { - timeout := time.NewTicker(2 * time.Minute) - for { - <-timeout.C - - api.mu.Lock() - for id, lastUsed := range api.lastUsed { - if time.Since(lastUsed).Seconds() >= filterTimeout { - delete(api.lastUsed, id) - if err := api.w.Unsubscribe(id); err != nil { - log.Error("could not unsubscribe whisper filter", "error", err) - } - log.Debug("delete whisper filter (timeout)", "id", id) - } - } - api.mu.Unlock() - } -} - // Version returns the Whisper sub-protocol version. func (api *PublicWhisperAPI) Version(ctx context.Context) string { return ProtocolVersionStr @@ -219,6 +196,19 @@ func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { return api.w.DeleteSymKey(id) } +// MakeLightClient turns the node into light client, which does not forward +// any incoming messages, and sends only messages originated in this node. +func (api *PublicWhisperAPI) MakeLightClient(ctx context.Context) bool { + api.w.lightClient = true + return api.w.lightClient +} + +// CancelLightClient cancels light client mode. +func (api *PublicWhisperAPI) CancelLightClient(ctx context.Context) bool { + api.w.lightClient = false + return !api.w.lightClient +} + //go:generate gencodec -type NewMessage -field-override newMessageOverride -out gen_newmessage_json.go // NewMessage represents a new whisper message that is posted through the RPC. @@ -292,7 +282,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er } // encrypt and sent message - whisperMsg, err := newSentMessage(params) + whisperMsg, err := NewSentMessage(params) if err != nil { return false, err } diff --git a/whisper/whisperv6/benchmarks_test.go b/whisper/whisperv6/benchmarks_test.go index 52c8f95ea6..0473179da5 100644 --- a/whisper/whisperv6/benchmarks_test.go +++ b/whisper/whisperv6/benchmarks_test.go @@ -39,7 +39,7 @@ func BenchmarkEncryptionSym(b *testing.B) { } for i := 0; i < b.N; i++ { - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Errorf("failed Wrap with seed %d: %s.", seed, err) @@ -64,7 +64,7 @@ func BenchmarkEncryptionAsym(b *testing.B) { params.Dst = &key.PublicKey for i := 0; i < b.N; i++ { - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -79,7 +79,7 @@ func BenchmarkDecryptionSymValid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -101,7 +101,7 @@ func BenchmarkDecryptionSymInvalid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -130,7 +130,7 @@ func BenchmarkDecryptionAsymValid(b *testing.B) { f := Filter{KeyAsym: key} params.KeySym = nil params.Dst = &key.PublicKey - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -157,7 +157,7 @@ func BenchmarkDecryptionAsymInvalid(b *testing.B) { } params.KeySym = nil params.Dst = &key.PublicKey - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -199,7 +199,7 @@ func BenchmarkPoW(b *testing.B) { for i := 0; i < b.N; i++ { increment(params.Payload) - msg, _ := newSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index 881945e9a8..c7bea2bb90 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -77,15 +77,19 @@ func NewEnvelope(ttl uint32, topic TopicType, msg *sentMessage) *Envelope { // Seal closes the envelope by spending the requested amount of time as a proof // of work on hashing the data. func (e *Envelope) Seal(options *MessageParams) error { - var target, bestBit int if options.PoW == 0 { - // adjust for the duration of Seal() execution only if execution time is predefined unconditionally + // PoW is not required + return nil + } + + var target, bestBit int + if options.PoW < 0 { + // target is not set - the function should run for a period + // of time specified in WorkTime param. Since we can predict + // the execution time, we can also adjust Expiry. e.Expiry += options.WorkTime } else { target = e.powToFirstBit(options.PoW) - if target < 1 { - target = 1 - } } buf := make([]byte, 64) @@ -143,7 +147,11 @@ func (e *Envelope) powToFirstBit(pow float64) int { x *= float64(e.TTL) bits := gmath.Log2(x) bits = gmath.Ceil(bits) - return int(bits) + res := int(bits) + if res < 1 { + res = 1 + } + return res } // Hash returns the SHA3 hash of the envelope, calculating it if not yet done. diff --git a/whisper/whisperv6/envelope_test.go b/whisper/whisperv6/envelope_test.go index 1ee1bec41b..410b250a3f 100644 --- a/whisper/whisperv6/envelope_test.go +++ b/whisper/whisperv6/envelope_test.go @@ -45,7 +45,7 @@ func TestEnvelopeOpenAcceptsOnlyOneKeyTypeInFilter(t *testing.T) { mrand.Read(params.Payload) - msg, err := newSentMessage(¶ms) + msg, err := NewSentMessage(¶ms) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index eb0c65fa3b..e4171f85c9 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -35,6 +35,7 @@ type Filter struct { PoW float64 // Proof of work as described in the Whisper spec AllowP2P bool // Indicates whether this filter is interested in direct peer-to-peer messages SymKeyHash common.Hash // The Keccak256Hash of the symmetric key, needed for optimization + id string // unique identifier Messages map[common.Hash]*ReceivedMessage mutex sync.RWMutex @@ -43,15 +44,21 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex + + topicMatcher map[TopicType]map[*Filter]struct{} // map a topic to the filters that are interested in being notified when a message matches that topic + allTopicsMatcher map[*Filter]struct{} // list all the filters that will be notified of a new message, no matter what its topic is + + whisper *Whisper + mutex sync.RWMutex } // NewFilters returns a newly created filter collection func NewFilters(w *Whisper) *Filters { return &Filters{ - watchers: make(map[string]*Filter), - whisper: w, + watchers: make(map[string]*Filter), + topicMatcher: make(map[TopicType]map[*Filter]struct{}), + allTopicsMatcher: make(map[*Filter]struct{}), + whisper: w, } } @@ -81,7 +88,9 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { watcher.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym) } + watcher.id = id fs.watchers[id] = watcher + fs.addTopicMatcher(watcher) return id, err } @@ -91,12 +100,51 @@ func (fs *Filters) Uninstall(id string) bool { fs.mutex.Lock() defer fs.mutex.Unlock() if fs.watchers[id] != nil { + fs.removeFromTopicMatchers(fs.watchers[id]) delete(fs.watchers, id) return true } return false } +// addTopicMatcher adds a filter to the topic matchers. +// If the filter's Topics array is empty, it will be tried on every topic. +// Otherwise, it will be tried on the topics specified. +func (fs *Filters) addTopicMatcher(watcher *Filter) { + if len(watcher.Topics) == 0 { + fs.allTopicsMatcher[watcher] = struct{}{} + } else { + for _, t := range watcher.Topics { + topic := BytesToTopic(t) + if fs.topicMatcher[topic] == nil { + fs.topicMatcher[topic] = make(map[*Filter]struct{}) + } + fs.topicMatcher[topic][watcher] = struct{}{} + } + } +} + +// removeFromTopicMatchers removes a filter from the topic matchers +func (fs *Filters) removeFromTopicMatchers(watcher *Filter) { + delete(fs.allTopicsMatcher, watcher) + for _, topic := range watcher.Topics { + delete(fs.topicMatcher[BytesToTopic(topic)], watcher) + } +} + +// getWatchersByTopic returns a slice containing the filters that +// match a specific topic +func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter { + res := make([]*Filter, 0, len(fs.allTopicsMatcher)) + for watcher := range fs.allTopicsMatcher { + res = append(res, watcher) + } + for watcher := range fs.topicMatcher[topic] { + res = append(res, watcher) + } + return res +} + // Get returns a filter from the collection with a specific ID func (fs *Filters) Get(id string) *Filter { fs.mutex.RLock() @@ -112,11 +160,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { fs.mutex.RLock() defer fs.mutex.RUnlock() - i := -1 // only used for logging info - for _, watcher := range fs.watchers { - i++ + candidates := fs.getWatchersByTopic(env.Topic) + for _, watcher := range candidates { if p2pMessage && !watcher.AllowP2P { - log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i)) + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcher.id)) continue } @@ -128,10 +175,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { if match { msg = env.Open(watcher) if msg == nil { - log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcher.id) } } else { - log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcher.id) } } @@ -194,16 +241,17 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) { // MatchMessage checks if the filter matches an already decrypted // message (i.e. a Message that has already been handled by -// MatchEnvelope when checked by a previous filter) +// MatchEnvelope when checked by a previous filter). +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { if f.PoW > 0 && msg.PoW < f.PoW { return false } if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() { - return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic) + return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) } else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() { - return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic) + return f.SymKeyHash == msg.SymKeyHash } return false } @@ -211,27 +259,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { // MatchEnvelope checks if it's worth decrypting the message. If // it returns `true`, client code is expected to attempt decrypting // the message and subsequently call MatchMessage. +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchEnvelope(envelope *Envelope) bool { - if f.PoW > 0 && envelope.pow < f.PoW { - return false - } - - return f.MatchTopic(envelope.Topic) -} - -// MatchTopic checks that the filter captures a given topic. -func (f *Filter) MatchTopic(topic TopicType) bool { - if len(f.Topics) == 0 { - // any topic matches - return true - } - - for _, bt := range f.Topics { - if matchSingleTopic(topic, bt) { - return true - } - } - return false + return f.PoW <= 0 || envelope.pow >= f.PoW } func matchSingleTopic(topic TopicType, bt []byte) bool { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index fc7db76712..0bb7986c39 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -199,7 +199,7 @@ func TestInstallIdenticalFilters(t *testing.T) { filter1.Src = ¶ms.Src.PublicKey filter2.Src = ¶ms.Src.PublicKey - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -303,10 +303,9 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.Topic[0] = 0xFF // ensure mismatch + params.Topic[0] = 0xFF // topic mismatch - // mismatch with pseudo-random data - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -314,20 +313,12 @@ func TestMatchEnvelope(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - match := fsym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) - } - match = fasym.MatchEnvelope(env) - if match { - t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) - } // encrypt symmetrically i := mrand.Int() % 4 fsym.Topics[i] = params.Topic[:] fasym.Topics[i] = params.Topic[:] - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -337,7 +328,7 @@ func TestMatchEnvelope(t *testing.T) { } // symmetric + matching topic: match - match = fsym.MatchEnvelope(env) + match := fsym.MatchEnvelope(env) if !match { t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) } @@ -372,7 +363,7 @@ func TestMatchEnvelope(t *testing.T) { } params.KeySym = nil params.Dst = &key.PublicKey - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -396,7 +387,7 @@ func TestMatchEnvelope(t *testing.T) { // asymmetric + matching topic: match fasym.Topics[i] = fasym.Topics[i+1] match = fasym.MatchEnvelope(env) - if match { + if !match { t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed) } @@ -431,7 +422,8 @@ func TestMatchEnvelope(t *testing.T) { // filter with topic + envelope without topic: mismatch fasym.Topics = fsym.Topics match = fasym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) } } @@ -453,7 +445,7 @@ func TestMatchMessageSym(t *testing.T) { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[index]) - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -487,7 +479,8 @@ func TestMatchMessageSym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -546,7 +539,7 @@ func TestMatchMessageAsym(t *testing.T) { keySymOrig := params.KeySym params.KeySym = nil - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -580,7 +573,8 @@ func TestMatchMessageAsym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -630,7 +624,7 @@ func generateCompatibeEnvelope(t *testing.T, f *Filter) *Envelope { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[2]) - sentMessage, err := newSentMessage(params) + sentMessage, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -806,7 +800,7 @@ func TestVariableTopics(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -829,8 +823,9 @@ func TestVariableTopics(t *testing.T) { f.Topics[i][lastTopicByte]++ match = f.MatchEnvelope(env) - if match { - t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers + t.Fatalf("MatchEnvelope symmetric with seed %d, step %d.", seed, i) } } } diff --git a/whisper/whisperv6/message.go b/whisper/whisperv6/message.go index 7def35f14f..b8318cbe8f 100644 --- a/whisper/whisperv6/message.go +++ b/whisper/whisperv6/message.go @@ -89,7 +89,7 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { } // NewSentMessage creates and initializes a non-signed, non-encrypted Whisper message. -func newSentMessage(params *MessageParams) (*sentMessage, error) { +func NewSentMessage(params *MessageParams) (*sentMessage, error) { const payloadSizeFieldMaxSize = 4 msg := sentMessage{} msg.Raw = make([]byte, 1, diff --git a/whisper/whisperv6/message_test.go b/whisper/whisperv6/message_test.go index 12a269f5db..0a5c1c8533 100644 --- a/whisper/whisperv6/message_test.go +++ b/whisper/whisperv6/message_test.go @@ -73,7 +73,7 @@ func singleMessageTest(t *testing.T, symmetric bool) { text := make([]byte, 0, 512) text = append(text, params.Payload...) - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -131,7 +131,7 @@ func TestMessageWrap(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -149,7 +149,7 @@ func TestMessageWrap(t *testing.T) { } // set PoW target too high, expect error - msg2, err := newSentMessage(params) + msg2, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -172,7 +172,7 @@ func TestMessageSeal(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -234,7 +234,7 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) { text := make([]byte, 0, 512) text = append(text, params.Payload...) - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -289,7 +289,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -303,7 +303,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -317,7 +317,7 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -335,7 +335,7 @@ func TestRlpEncode(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -379,7 +379,7 @@ func singlePaddingTest(t *testing.T, padSize int) { if n != padSize { t.Fatalf("padding is not copied (seed %d): %s", seed, err) } - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 4f9a7c3780..6d75290fde 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -19,6 +19,7 @@ package whisperv6 import ( "fmt" "math" + "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -36,7 +37,9 @@ type Peer struct { trusted bool powRequirement float64 - bloomFilter []byte // may contain nil in case of full node + bloomMu sync.Mutex + bloomFilter []byte + fullNode bool known *set.Set // Messages already known by the peer to avoid wasting bandwidth @@ -53,6 +56,8 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { powRequirement: 0.0, known: set.New(), quit: make(chan struct{}), + bloomFilter: makeFullNodeBloom(), + fullNode: true, } } @@ -118,11 +123,7 @@ func (peer *Peer) handshake() error { if sz != bloomFilterSize && sz != 0 { return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", peer.ID(), sz) } - if isFullNode(bloom) { - peer.bloomFilter = nil - } else { - peer.bloomFilter = bloom - } + peer.setBloomFilter(bloom) } } @@ -226,10 +227,25 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error { } func (peer *Peer) bloomMatch(env *Envelope) bool { - if peer.bloomFilter == nil { - // no filter - full node, accepts all envelops - return true - } - - return bloomFilterMatch(peer.bloomFilter, env.Bloom()) + peer.bloomMu.Lock() + defer peer.bloomMu.Unlock() + return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom()) +} + +func (peer *Peer) setBloomFilter(bloom []byte) { + peer.bloomMu.Lock() + defer peer.bloomMu.Unlock() + peer.bloomFilter = bloom + peer.fullNode = isFullNode(bloom) + if peer.fullNode && peer.bloomFilter == nil { + peer.bloomFilter = makeFullNodeBloom() + } +} + +func makeFullNodeBloom() []byte { + bloom := make([]byte, bloomFilterSize) + for i := 0; i < bloomFilterSize; i++ { + bloom[i] = 0xFF + } + return bloom } diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 17f70129b5..6d7754d790 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -23,6 +23,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "testing" "time" @@ -70,9 +71,8 @@ var keys = []string{ "7184c1701569e3a4c4d2ddce691edd983b81e42e09196d332e1ae2f1e062cff4", } -const NumNodes = 16 // must not exceed the number of keys (32) - type TestData struct { + started int64 counter [NumNodes]int mutex sync.RWMutex } @@ -84,21 +84,29 @@ type TestNode struct { filerID string } +const NumNodes = 8 // must not exceed the number of keys (32) + var result TestData var nodes [NumNodes]*TestNode var sharedKey = hexutil.MustDecode("0x03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31") +var wrongKey = hexutil.MustDecode("0xf91156714d7ec88d3edc1c652c2181dbb3044e8771c683f3b30d33c12b986b11") var sharedTopic = TopicType{0xF, 0x1, 0x2, 0} -var expectedMessage = []byte("per rectum ad astra") +var wrongTopic = TopicType{0, 0, 0, 0} +var expectedMessage = []byte("per aspera ad astra") +var unexpectedMessage = []byte("per rectum ad astra") var masterBloomFilter []byte var masterPow = 0.00000001 var round = 1 +var debugMode = false +var prevTime time.Time +var cntPrev int func TestSimulation(t *testing.T) { // create a chain of whisper nodes, // installs the filters with shared (predefined) parameters initialize(t) - // each node sends a number of random (undecryptable) messages + // each node sends one random (not decryptable) message for i := 0; i < NumNodes; i++ { sendMsg(t, false, i) } @@ -115,7 +123,6 @@ func TestSimulation(t *testing.T) { // send new pow and bloom exchange messages resetParams(t) - round++ // node #1 sends one expected (decryptable) message sendMsg(t, true, 1) @@ -140,6 +147,8 @@ func resetParams(t *testing.T) { for i := 0; i < NumNodes; i++ { nodes[i].shh.SetBloomFilter(masterBloomFilter) } + + round++ } func initBloom(t *testing.T) { @@ -219,15 +228,20 @@ func initialize(t *testing.T) { nodes[i] = &node } - for i := 1; i < NumNodes; i++ { - go nodes[i].server.Start() + for i := 0; i < NumNodes; i++ { + go startServer(t, nodes[i].server) } - // we need to wait until the first node actually starts - err = nodes[0].server.Start() + waitForServersToStart(t) +} + +func startServer(t *testing.T, s *p2p.Server) { + err := s.Start() if err != nil { t.Fatalf("failed to start the fisrt server.") } + + atomic.AddInt64(&result.started, 1) } func stopServers() { @@ -246,8 +260,10 @@ func checkPropagation(t *testing.T, includingNodeZero bool) { return } - const cycle = 50 - const iterations = 200 + prevTime = time.Now() + // (cycle * iterations) should not exceed 50 seconds, since TTL=50 + const cycle = 200 // time in milliseconds + const iterations = 250 first := 0 if !includingNodeZero { @@ -262,29 +278,29 @@ func checkPropagation(t *testing.T, includingNodeZero bool) { } mail := f.Retrieve() - if !validateMail(t, i, mail) { - return - } + validateMail(t, i, mail) if isTestComplete() { + checkTestStatus() return } } + checkTestStatus() time.Sleep(cycle * time.Millisecond) } - t.Fatalf("Test was not complete: timeout %d seconds.", iterations*cycle/1000) - if !includingNodeZero { f := nodes[0].shh.GetFilter(nodes[0].filerID) if f != nil { t.Fatalf("node zero received a message with low PoW.") } } + + t.Fatalf("Test was not complete (%d round): timeout %d seconds. nodes=%v", round, iterations*cycle/1000, nodes) } -func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool { +func validateMail(t *testing.T, index int, mail []*ReceivedMessage) { var cnt int for _, m := range mail { if bytes.Equal(m.Payload, expectedMessage) { @@ -294,14 +310,13 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool { if cnt == 0 { // no messages received yet: nothing is wrong - return true + return } if cnt > 1 { t.Fatalf("node %d received %d.", index, cnt) - return false } - if cnt > 0 { + if cnt == 1 { result.mutex.Lock() defer result.mutex.Unlock() result.counter[index] += cnt @@ -309,7 +324,28 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool { t.Fatalf("node %d accumulated %d.", index, result.counter[index]) } } - return true +} + +func checkTestStatus() { + var cnt int + var arr [NumNodes]int + + for i := 0; i < NumNodes; i++ { + arr[i] = nodes[i].server.PeerCount() + envelopes := nodes[i].shh.Envelopes() + if len(envelopes) >= NumNodes { + cnt++ + } + } + + if debugMode { + if cntPrev != cnt { + fmt.Printf(" %v \t number of nodes that have received all msgs: %d, number of peers per node: %v \n", + time.Since(prevTime), cnt, arr) + prevTime = time.Now() + cntPrev = cnt + } + } } func isTestComplete() bool { @@ -324,7 +360,7 @@ func isTestComplete() bool { for i := 0; i < NumNodes; i++ { envelopes := nodes[i].shh.Envelopes() - if len(envelopes) < 2 { + if len(envelopes) < NumNodes+1 { return false } } @@ -339,12 +375,13 @@ func sendMsg(t *testing.T, expected bool, id int) { opt := MessageParams{KeySym: sharedKey, Topic: sharedTopic, Payload: expectedMessage, PoW: 0.00000001, WorkTime: 1} if !expected { - opt.KeySym[0]++ - opt.Topic[0]++ - opt.Payload = opt.Payload[1:] + opt.KeySym = wrongKey + opt.Topic = wrongTopic + opt.Payload = unexpectedMessage + opt.Payload[0] = byte(id) } - msg, err := newSentMessage(&opt) + msg, err := NewSentMessage(&opt) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -368,7 +405,7 @@ func TestPeerBasic(t *testing.T) { } params.PoW = 0.001 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -434,7 +471,10 @@ func checkPowExchange(t *testing.T) { func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool { for i, node := range nodes { for peer := range node.shh.peers { - if !bytes.Equal(peer.bloomFilter, masterBloomFilter) { + peer.bloomMu.Lock() + equals := bytes.Equal(peer.bloomFilter, masterBloomFilter) + peer.bloomMu.Unlock() + if !equals { if mustPass { t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got", i, round, masterBloomFilter, peer.bloomFilter) @@ -459,3 +499,16 @@ func checkBloomFilterExchange(t *testing.T) { time.Sleep(50 * time.Millisecond) } } + +func waitForServersToStart(t *testing.T) { + const iterations = 200 + var started int64 + for j := 0; j < iterations; j++ { + time.Sleep(50 * time.Millisecond) + started = atomic.LoadInt64(&result.started) + if started == NumNodes { + return + } + } + t.Fatalf("Failed to start all the servers, running: %d", started) +} diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index d75ad04ac3..81f59b3cc7 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -82,6 +82,8 @@ type Whisper struct { syncAllowance int // maximum time in seconds allowed to process the whisper-related messages + lightClient bool // indicates is this node is pure light client (does not forward any messages) + statsMu sync.Mutex // guard stats stats Statistics // Statistics of whisper node @@ -587,11 +589,8 @@ func (whisper *Whisper) Unsubscribe(id string) error { // Send injects a message into the whisper send queue, to be distributed in the // network in the coming cycles. func (whisper *Whisper) Send(envelope *Envelope) error { - ok, err := whisper.add(envelope) - if err != nil { - return err - } - if !ok { + ok, err := whisper.add(envelope, false) + if err == nil && !ok { return fmt.Errorf("failed to add envelope") } return err @@ -673,7 +672,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { trouble := false for _, env := range envelopes { - cached, err := whisper.add(env) + cached, err := whisper.add(env, whisper.lightClient) if err != nil { trouble = true log.Error("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) @@ -710,11 +709,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("failed to decode bloom filter exchange message, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid bloom filter exchange message") } - if isFullNode(bloom) { - p.bloomFilter = nil - } else { - p.bloomFilter = bloom - } + p.setBloomFilter(bloom) case p2pMessageCode: // peer-to-peer message, sent directly to peer bypassing PoW checks, etc. // this message is not supposed to be forwarded to other peers, and @@ -750,7 +745,8 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { // add inserts a new envelope into the message pool to be distributed within the // whisper network. It also inserts the envelope into the expiration pool at the // appropriate time-stamp. In case of error, connection should be dropped. -func (whisper *Whisper) add(envelope *Envelope) (bool, error) { +// param isP2P indicates whether the message is peer-to-peer (should not be forwarded). +func (whisper *Whisper) add(envelope *Envelope, isP2P bool) (bool, error) { now := uint32(time.Now().Unix()) sent := envelope.Expiry - envelope.TTL @@ -815,7 +811,7 @@ func (whisper *Whisper) add(envelope *Envelope) (bool, error) { whisper.statsMu.Lock() whisper.stats.memoryUsed += envelope.size() whisper.statsMu.Unlock() - whisper.postEvent(envelope, false) // notify the local node about the new message + whisper.postEvent(envelope, isP2P) // notify the local node about the new message if whisper.mailServer != nil { whisper.mailServer.Archive(envelope) } @@ -932,24 +928,6 @@ func (whisper *Whisper) Envelopes() []*Envelope { return all } -// Messages iterates through all currently floating envelopes -// and retrieves all the messages, that this filter could decrypt. -func (whisper *Whisper) Messages(id string) []*ReceivedMessage { - result := make([]*ReceivedMessage, 0) - whisper.poolMu.RLock() - defer whisper.poolMu.RUnlock() - - if filter := whisper.filters.Get(id); filter != nil { - for _, env := range whisper.envelopes { - msg := filter.processEnvelope(env) - if msg != nil { - result = append(result, msg) - } - } - } - return result -} - // isEnvelopeCached checks if envelope with specific hash has already been received and cached. func (whisper *Whisper) isEnvelopeCached(hash common.Hash) bool { whisper.poolMu.Lock() @@ -1049,7 +1027,6 @@ func isFullNode(bloom []byte) bool { func bloomFilterMatch(filter, sample []byte) bool { if filter == nil { - // full node, accepts all messages return true } diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 838cb7b851..3a219cd133 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -75,10 +75,6 @@ func TestWhisperBasic(t *testing.T) { if len(mail) != 0 { t.Fatalf("failed w.Envelopes().") } - m := w.Messages("non-existent") - if len(m) != 0 { - t.Fatalf("failed w.Messages.") - } derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New) if !validateDataIntegrity(derived, aesKeyLength) { @@ -471,7 +467,7 @@ func TestExpiry(t *testing.T) { } params.TTL = 1 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -537,7 +533,7 @@ func TestCustomization(t *testing.T) { params.Topic = BytesToTopic(f.Topics[2]) params.PoW = smallPoW params.TTL = 3600 * 24 // one day - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -558,7 +554,7 @@ func TestCustomization(t *testing.T) { } params.TTL++ - msg, err = newSentMessage(params) + msg, err = NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -593,7 +589,7 @@ func TestCustomization(t *testing.T) { } // check w.messages() - id, err := w.Subscribe(f) + _, err = w.Subscribe(f) if err != nil { t.Fatalf("failed subscribe with seed %d: %s.", seed, err) } @@ -602,11 +598,6 @@ func TestCustomization(t *testing.T) { if len(mail) > 0 { t.Fatalf("received premature mail") } - - mail = w.Messages(id) - if len(mail) != 2 { - t.Fatalf("failed to get whisper messages") - } } func TestSymmetricSendCycle(t *testing.T) { @@ -647,7 +638,7 @@ func TestSymmetricSendCycle(t *testing.T) { params.PoW = filter1.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -725,7 +716,7 @@ func TestSymmetricSendWithoutAKey(t *testing.T) { params.PoW = filter.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) } @@ -791,7 +782,7 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { params.PoW = filter.PoW params.WorkTime = 10 params.TTL = 50 - msg, err := newSentMessage(params) + msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) }