Merge branch 'master' into eth_getBlockReceipts

README.md

@@ -9,28 +9,30 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
-Automated builds are available for stable releases and the unstable master branch.
+Automated builds are available for stable releases and the unstable master branch. Binary
-Binary archives are published at https://geth.ethereum.org/downloads/.
+archives are published at https://geth.ethereum.org/downloads/.
 
 ## Building the source
 
-For prerequisites and detailed build instructions please read the
+For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki.
-[Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
-on the wiki.
 
-Building geth requires both a Go (version 1.10 or later) and a C compiler.
+Building `geth` requires both a Go (version 1.10 or later) and a C compiler. You can install
-You can install them using your favourite package manager.
+them using your favourite package manager. Once the dependencies are installed, run
-Once the dependencies are installed, run
 
-    make geth
+```shell
+make geth
+```
 
 or, to build the full suite of utilities:
 
-    make all
+```shell
+make all
+```
 
 ## Executables
 
-The go-ethereum project comes with several wrappers/executables found in the `cmd` directory.
+The go-ethereum project comes with several wrappers/executables found in the `cmd`
+directory.
 
 |    Command    | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 | :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -42,269 +44,301 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 |   `rlpdump`   | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`).                                                                                                                                                                                                                                 |
 |   `puppeth`   | a CLI wizard that aids in creating a new Ethereum network.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 
-## Running geth
+## Running `geth`
 
 Going through all the possible command line flags is out of scope here (please consult our
-[CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options)), but we've
+[CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options)),
-enumerated a few common parameter combos to get you up to speed quickly on how you can run your
+but we've enumerated a few common parameter combos to get you up to speed quickly
-own Geth instance.
+on how you can run your own `geth` instance.
 
 ### Full node on the main Ethereum network
 
-By far the most common scenario is people wanting to simply interact with the Ethereum network:
+By far the most common scenario is people wanting to simply interact with the Ethereum
-create accounts; transfer funds; deploy and interact with contracts. For this particular use-case
+network: create accounts; transfer funds; deploy and interact with contracts. For this
-the user doesn't care about years-old historical data, so we can fast-sync quickly to the current
+particular use-case the user doesn't care about years-old historical data, so we can
-state of the network. To do so:
+fast-sync quickly to the current state of the network. To do so:
 
-```
+```shell
 $ geth console
 ```
 
 This command will:
-
+ * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag),
- * Start geth in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to
+   causing it to download more data in exchange for avoiding processing the entire history
-   download more data in exchange for avoiding processing the entire history of the Ethereum network,
+   of the Ethereum network, which is very CPU intensive.
-   which is very CPU intensive.
+ * Start up `geth`'s built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
- * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
-   as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
+   as well as `geth`'s own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
-   This tool is optional and if you leave it out you can always attach to an already running Geth instance
+   This tool is optional and if you leave it out you can always attach to an already running
-   with `geth attach`.
+   `geth` instance with `geth attach`.
 
 ### A Full node on the Ethereum test network
 
-Transitioning towards developers, if you'd like to play around with creating Ethereum contracts, you
+Transitioning towards developers, if you'd like to play around with creating Ethereum
-almost certainly would like to do that without any real money involved until you get the hang of the
+contracts, you almost certainly would like to do that without any real money involved until
-entire system. In other words, instead of attaching to the main network, you want to join the **test**
+you get the hang of the entire system. In other words, instead of attaching to the main
-network with your node, which is fully equivalent to the main network, but with play-Ether only.
+network, you want to join the **test** network with your node, which is fully equivalent to
+the main network, but with play-Ether only.
 
-```
+```shell
 $ geth --testnet console
 ```
 
-The `console` subcommand has the exact same meaning as above and they are equally useful on the
+The `console` subcommand has the exact same meaning as above and they are equally
-testnet too. Please see above for their explanations if you've skipped here.
+useful on the testnet too. Please see above for their explanations if you've skipped here.
 
-Specifying the `--testnet` flag, however, will reconfigure your Geth instance a bit:
+Specifying the `--testnet` flag, however, will reconfigure your `geth` instance a bit:
 
- * Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
+ * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth`
-   itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
+   will nest itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on
-   and Linux this also means that attaching to a running testnet node requires the use of a custom
+   Linux). Note, on OSX and Linux this also means that attaching to a running testnet node
-   endpoint since `geth attach` will try to attach to a production node endpoint by default. E.g.
+   requires the use of a custom endpoint since `geth attach` will try to attach to a
-   `geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by this.
+   production node endpoint by default. E.g.
- * Instead of connecting the main Ethereum network, the client will connect to the test network,
+   `geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by
-   which uses different P2P bootnodes, different network IDs and genesis states.
+   this.
-
+ * Instead of connecting the main Ethereum network, the client will connect to the test
-*Note: Although there are some internal protective measures to prevent transactions from crossing
+   network, which uses different P2P bootnodes, different network IDs and genesis states.
-over between the main network and test network, you should make sure to always use separate accounts
+   
-for play-money and real-money. Unless you manually move accounts, Geth will by default correctly
+*Note: Although there are some internal protective measures to prevent transactions from
-separate the two networks and will not make any accounts available between them.*
+crossing over between the main network and test network, you should make sure to always
+use separate accounts for play-money and real-money. Unless you manually move
+accounts, `geth` will by default correctly separate the two networks and will not make any
+accounts available between them.*
 
 ### Full node on the Rinkeby test network
 
-The above test network is a cross-client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty/security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
+The above test network is a cross-client one based on the ethash proof-of-work consensus
+algorithm. As such, it has certain extra overhead and is more susceptible to reorganization
+attacks due to the network's low difficulty/security. Go Ethereum also supports connecting
+to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io)
+(operated by members of the community). This network is lighter, more secure, but is only
+supported by go-ethereum.
 
-```
+```shell
 $ geth --rinkeby console
 ```
 
 ### Configuration
 
-As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via:
+As an alternative to passing the numerous flags to the `geth` binary, you can also pass a
+configuration file via:
 
-```
+```shell
 $ geth --config /path/to/your_config.toml
 ```
 
-To get an idea how the file should look like you can use the `dumpconfig` subcommand to export your existing configuration:
+To get an idea how the file should look like you can use the `dumpconfig` subcommand to
+export your existing configuration:
 
-```
+```shell
 $ geth --your-favourite-flags dumpconfig
 ```
 
-*Note: This works only with geth v1.6.0 and above.*
+*Note: This works only with `geth` v1.6.0 and above.*
 
 #### Docker quick start
 
-One of the quickest ways to get Ethereum up and running on your machine is by using Docker:
+One of the quickest ways to get Ethereum up and running on your machine is by using
+Docker:
 
-```
+```shell
 docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
            -p 8545:8545 -p 30303:30303 \
            ethereum/client-go
 ```
 
-This will start geth in fast-sync mode with a DB memory allowance of 1GB just as the above command does.  It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
+This will start `geth` in fast-sync mode with a DB memory allowance of 1GB just as the
+above command does.  It will also create a persistent volume in your home directory for
+saving your blockchain as well as map the default ports. There is also an `alpine` tag
+available for a slim version of the image.
 
-Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
+Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers
+and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not
+accessible from the outside.
 
-### Programmatically interfacing Geth nodes
+### Programmatically interfacing `geth` nodes
 
-As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
+As a developer, sooner rather than later you'll want to start interacting with `geth` and the
-network via your own programs and not manually through the console. To aid this, Geth has built-in
+Ethereum network via your own programs and not manually through the console. To aid
-support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
+this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC)
-[Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
+and [`geth` specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)).
-exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows).
+These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based
+platforms, and named pipes on Windows).
 
-The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
+The IPC interface is enabled by default and exposes all the APIs supported by `geth`,
-and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
+whereas the HTTP and WS interfaces need to manually be enabled and only expose a
-These can be turned on/off and configured as you'd expect.
+subset of APIs due to security reasons. These can be turned on/off and configured as
+you'd expect.
 
 HTTP based JSON-RPC API options:
 
   * `--rpc` Enable the HTTP-RPC server
-  * `--rpcaddr` HTTP-RPC server listening interface (default: "localhost")
+  * `--rpcaddr` HTTP-RPC server listening interface (default: `localhost`)
-  * `--rpcport` HTTP-RPC server listening port (default: 8545)
+  * `--rpcport` HTTP-RPC server listening port (default: `8545`)
-  * `--rpcapi` API's offered over the HTTP-RPC interface (default: "eth,net,web3")
+  * `--rpcapi` API's offered over the HTTP-RPC interface (default: `eth,net,web3`)
   * `--rpccorsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced)
   * `--ws` Enable the WS-RPC server
-  * `--wsaddr` WS-RPC server listening interface (default: "localhost")
+  * `--wsaddr` WS-RPC server listening interface (default: `localhost`)
-  * `--wsport` WS-RPC server listening port (default: 8546)
+  * `--wsport` WS-RPC server listening port (default: `8546`)
-  * `--wsapi` API's offered over the WS-RPC interface (default: "eth,net,web3")
+  * `--wsapi` API's offered over the WS-RPC interface (default: `eth,net,web3`)
   * `--wsorigins` Origins from which to accept websockets requests
   * `--ipcdisable` Disable the IPC-RPC server
-  * `--ipcapi` API's offered over the IPC-RPC interface (default: "admin,debug,eth,miner,net,personal,shh,txpool,web3")
+  * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`)
   * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
-You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect
+You'll need to use your own programming environments' capabilities (libraries, tools, etc) to
-via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification)
+connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll
-on all transports. You can reuse the same connection for multiple requests!
+need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You
+can reuse the same connection for multiple requests!
 
-**Note: Please understand the security implications of opening up an HTTP/WS based transport before
+**Note: Please understand the security implications of opening up an HTTP/WS based
-doing so! Hackers on the internet are actively trying to subvert Ethereum nodes with exposed APIs!
+transport before doing so! Hackers on the internet are actively trying to subvert
-Further, all browser tabs can access locally running web servers, so malicious web pages could try to
+Ethereum nodes with exposed APIs! Further, all browser tabs can access locally
-subvert locally available APIs!**
+running web servers, so malicious web pages could try to subvert locally available
+APIs!**
 
 ### Operating a private network
 
-Maintaining your own private network is more involved as a lot of configurations taken for granted in
+Maintaining your own private network is more involved as a lot of configurations taken for
-the official networks need to be manually set up.
+granted in the official networks need to be manually set up.
 
 #### Defining the private genesis state
 
-First, you'll need to create the genesis state of your networks, which all nodes need to be aware of
+First, you'll need to create the genesis state of your networks, which all nodes need to be
-and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`):
+aware of and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`):
 
 ```json
 {
   "config": {
-        "chainId": 0,
+    "chainId": 0,
-        "homesteadBlock": 0,
+    "homesteadBlock": 0,
-        "eip155Block": 0,
+    "eip155Block": 0,
-        "eip158Block": 0
+    "eip158Block": 0
-    },
+  },
-  "alloc"      : {},
+  "alloc": {},
-  "coinbase"   : "0x0000000000000000000000000000000000000000",
+  "coinbase": "0x0000000000000000000000000000000000000000",
-  "difficulty" : "0x20000",
+  "difficulty": "0x20000",
-  "extraData"  : "",
+  "extraData": "",
-  "gasLimit"   : "0x2fefd8",
+  "gasLimit": "0x2fefd8",
-  "nonce"      : "0x0000000000000042",
+  "nonce": "0x0000000000000042",
-  "mixhash"    : "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000",
-  "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
-  "timestamp"  : "0x00"
+  "timestamp": "0x00"
 }
 ```
 
-The above fields should be fine for most purposes, although we'd recommend changing the `nonce` to
+The above fields should be fine for most purposes, although we'd recommend changing
-some random value so you prevent unknown remote nodes from being able to connect to you. If you'd
+the `nonce` to some random value so you prevent unknown remote nodes from being able
-like to pre-fund some accounts for easier testing, you can populate the `alloc` field with account
+to connect to you. If you'd like to pre-fund some accounts for easier testing, you can
-configs:
+populate the `alloc` field with account configs:
 
 ```json
 "alloc": {
-  "0x0000000000000000000000000000000000000001": {"balance": "111111111"},
+  "0x0000000000000000000000000000000000000001": {
-  "0x0000000000000000000000000000000000000002": {"balance": "222222222"}
+    "balance": "111111111"
+  },
+  "0x0000000000000000000000000000000000000002": {
+    "balance": "222222222"
+  }
 }
 ```
 
-With the genesis state defined in the above JSON file, you'll need to initialize **every** Geth node
+With the genesis state defined in the above JSON file, you'll need to initialize **every**
-with it prior to starting it up to ensure all blockchain parameters are correctly set:
+`geth` node with it prior to starting it up to ensure all blockchain parameters are correctly
+set:
 
-```
+```shell
 $ geth init path/to/genesis.json
 ```
 
 #### Creating the rendezvous point
 
-With all nodes that you want to run initialized to the desired genesis state, you'll need to start a
+With all nodes that you want to run initialized to the desired genesis state, you'll need to
-bootstrap node that others can use to find each other in your network and/or over the internet. The
+start a bootstrap node that others can use to find each other in your network and/or over
-clean way is to configure and run a dedicated bootnode:
+the internet. The clean way is to configure and run a dedicated bootnode:
 
-```
+```shell
 $ bootnode --genkey=boot.key
 $ bootnode --nodekey=boot.key
 ```
 
 With the bootnode online, it will display an [`enode` URL](https://github.com/ethereum/wiki/wiki/enode-url-format)
-that other nodes can use to connect to it and exchange peer information. Make sure to replace the
+that other nodes can use to connect to it and exchange peer information. Make sure to
-displayed IP address information (most probably `[::]`) with your externally accessible IP to get the
+replace the displayed IP address information (most probably `[::]`) with your externally
-actual `enode` URL.
+accessible IP to get the actual `enode` URL.
 
-*Note: You could also use a full-fledged Geth node as a bootnode, but it's the less recommended way.*
+*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less
+recommended way.*
 
 #### Starting up your member nodes
 
-With the bootnode operational and externally reachable (you can try `telnet <ip> <port>` to ensure
+With the bootnode operational and externally reachable (you can try
-it's indeed reachable), start every subsequent Geth node pointed to the bootnode for peer discovery
+`telnet <ip> <port>` to ensure it's indeed reachable), start every subsequent `geth`
-via the `--bootnodes` flag. It will probably also be desirable to keep the data directory of your
+node pointed to the bootnode for peer discovery via the `--bootnodes` flag. It will
-private network separated, so do also specify a custom `--datadir` flag.
+probably also be desirable to keep the data directory of your private network separated, so
+do also specify a custom `--datadir` flag.
 
-```
+```shell
 $ geth --datadir=path/to/custom/data/folder --bootnodes=<bootnode-enode-url-from-above>
 ```
 
-*Note: Since your network will be completely cut off from the main and test networks, you'll also
+*Note: Since your network will be completely cut off from the main and test networks, you'll
-need to configure a miner to process transactions and create new blocks for you.*
+also need to configure a miner to process transactions and create new blocks for you.*
 
 #### Running a private miner
 
-Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, requiring
+Mining on the public Ethereum network is a complex task as it's only feasible using GPUs,
-an OpenCL or CUDA enabled `ethminer` instance. For information on such a setup, please consult the
+requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a
-[EtherMining subreddit](https://www.reddit.com/r/EtherMining/) and the [Genoil miner](https://github.com/Genoil/cpp-ethereum)
+setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/)
-repository.
+and the [Genoil miner](https://github.com/Genoil/cpp-ethereum) repository.
 
-In a private network setting, however a single CPU miner instance is more than enough for practical
+In a private network setting, however a single CPU miner instance is more than enough for
-purposes as it can produce a stable stream of blocks at the correct intervals without needing heavy
+practical purposes as it can produce a stable stream of blocks at the correct intervals
-resources (consider running on a single thread, no need for multiple ones either). To start a Geth
+without needing heavy resources (consider running on a single thread, no need for multiple
-instance for mining, run it with all your usual flags, extended by:
+ones either). To start a `geth` instance for mining, run it with all your usual flags, extended
+by:
 
-```
+```shell
 $ geth <usual-flags> --mine --minerthreads=1 --etherbase=0x0000000000000000000000000000000000000000
 ```
 
-Which will start mining blocks and transactions on a single CPU thread, crediting all proceedings to
+Which will start mining blocks and transactions on a single CPU thread, crediting all
-the account specified by `--etherbase`. You can further tune the mining by changing the default gas
+proceedings to the account specified by `--etherbase`. You can further tune the mining
-limit blocks converge to (`--targetgaslimit`) and the price transactions are accepted at (`--gasprice`).
+by changing the default gas limit blocks converge to (`--targetgaslimit`) and the price
+transactions are accepted at (`--gasprice`).
 
 ## Contribution
 
-Thank you for considering to help out with the source code! We welcome contributions from
+Thank you for considering to help out with the source code! We welcome contributions
-anyone on the internet, and are grateful for even the smallest of fixes!
+from anyone on the internet, and are grateful for even the smallest of fixes!
 
 If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request
-for the maintainers to review and merge into the main code base. If you wish to submit more
+for the maintainers to review and merge into the main code base. If you wish to submit
-complex changes though, please check up with the core devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum)
+more complex changes though, please check up with the core devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum)
-to ensure those changes are in line with the general philosophy of the project and/or get some
+to ensure those changes are in line with the general philosophy of the project and/or get
-early feedback which can make both your efforts much lighter as well as our review and merge
+some early feedback which can make both your efforts much lighter as well as our review
-procedures quick and simple.
+and merge procedures quick and simple.
 
 Please make sure your contributions adhere to our coding guidelines:
 
- * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting)
- * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+   guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary)
+   guidelines.
  * Pull requests need to be based on and opened against the `master` branch.
  * Commit messages should be prefixed with the package(s) they modify.
    * E.g. "eth, rpc: make trace configs optional"
 
 Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, managing project dependencies, and testing procedures.
+for more details on configuring your environment, managing project dependencies, and
+testing procedures.
 
 ## License
 
 The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the
-[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), also
+[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html),
-included in our repository in the `COPYING.LESSER` file.
+also included in our repository in the `COPYING.LESSER` file.
 
 The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the
-[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also included
+[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also
-in our repository in the `COPYING` file.
+included in our repository in the `COPYING` file.

accounts/abi/bind/auth.go

@@ -43,9 +43,9 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	return NewKeyedTransactor(key.PrivateKey), nil
 }
 
-// NewTransactor is a utility method to easily create a transaction signer from
+// NewKeystoreTransactor is a utility method to easily create a transaction signer from
 // an decrypted key from a keystore
-func NewTransactorFromKeyStore(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) {
+func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) {
 	return &TransactOpts{
 		From: account.Address,
 		Signer: func(signer types.Signer, address common.Address, tx *types.Transaction) (*types.Transaction, error) {

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.12.5.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://dl.google.com/go/go1.12.6.windows-%GETH_ARCH%.zip
-  - 7z x go1.12.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.12.6.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

core/blockchain.go

@@ -46,6 +46,10 @@ import (
 )
 
 var (
+	headBlockGauge     = metrics.NewRegisteredGauge("chain/head/block", nil)
+	headHeaderGauge    = metrics.NewRegisteredGauge("chain/head/header", nil)
+	headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil)
+
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)
 	accountHashTimer   = metrics.NewRegisteredTimer("chain/account/hashes", nil)
 	accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil)
@@ -332,6 +336,7 @@ func (bc *BlockChain) loadLastState() error {
 	}
 	// Everything seems to be fine, set as the head block
 	bc.currentBlock.Store(currentBlock)
+	headBlockGauge.Update(int64(currentBlock.NumberU64()))
 
 	// Restore the last known head header
 	currentHeader := currentBlock.Header()
@@ -344,12 +349,14 @@ func (bc *BlockChain) loadLastState() error {
 
 	// Restore the last known head fast block
 	bc.currentFastBlock.Store(currentBlock)
+	headFastBlockGauge.Update(int64(currentBlock.NumberU64()))
+
 	if head := rawdb.ReadHeadFastBlockHash(bc.db); head != (common.Hash{}) {
 		if block := bc.GetBlockByHash(head); block != nil {
 			bc.currentFastBlock.Store(block)
+			headFastBlockGauge.Update(int64(block.NumberU64()))
 		}
 	}
-
 	// Issue a status log for the user
 	currentFastBlock := bc.CurrentFastBlock()
 
@@ -388,6 +395,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 			}
 			rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash())
 			bc.currentBlock.Store(newHeadBlock)
+			headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
 		}
 
 		// Rewind the fast block in a simpleton way to the target head
@@ -399,6 +407,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 			}
 			rawdb.WriteHeadFastBlockHash(db, newHeadFastBlock.Hash())
 			bc.currentFastBlock.Store(newHeadFastBlock)
+			headFastBlockGauge.Update(int64(newHeadFastBlock.NumberU64()))
 		}
 	}
 
@@ -450,6 +459,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	// If all checks out, manually set the head block
 	bc.chainmu.Lock()
 	bc.currentBlock.Store(block)
+	headBlockGauge.Update(int64(block.NumberU64()))
 	bc.chainmu.Unlock()
 
 	log.Info("Committed new head block", "number", block.Number(), "hash", hash)
@@ -522,9 +532,12 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	bc.genesisBlock = genesis
 	bc.insert(bc.genesisBlock)
 	bc.currentBlock.Store(bc.genesisBlock)
+	headBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
+
 	bc.hc.SetGenesis(bc.genesisBlock.Header())
 	bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
 	bc.currentFastBlock.Store(bc.genesisBlock)
+	headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
 
 	return nil
 }
@@ -598,6 +611,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 	rawdb.WriteHeadBlockHash(bc.db, block.Hash())
 
 	bc.currentBlock.Store(block)
+	headBlockGauge.Update(int64(block.NumberU64()))
 
 	// If the block is better than our head or is on a different chain, force update heads
 	if updateHeads {
@@ -605,6 +619,7 @@ func (bc *BlockChain) insert(block *types.Block) {
 		rawdb.WriteHeadFastBlockHash(bc.db, block.Hash())
 
 		bc.currentFastBlock.Store(block)
+		headFastBlockGauge.Update(int64(block.NumberU64()))
 	}
 }
 
@@ -862,11 +877,13 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
 			newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1)
 			rawdb.WriteHeadFastBlockHash(bc.db, newFastBlock.Hash())
 			bc.currentFastBlock.Store(newFastBlock)
+			headFastBlockGauge.Update(int64(newFastBlock.NumberU64()))
 		}
 		if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash {
 			newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1)
 			rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash())
 			bc.currentBlock.Store(newBlock)
+			headBlockGauge.Update(int64(newBlock.NumberU64()))
 		}
 	}
 	// Truncate ancient data which exceeds the current header.
@@ -952,6 +969,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 {
 				rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
 				bc.currentFastBlock.Store(head)
+				headFastBlockGauge.Update(int64(head.NumberU64()))
 				isCanonical = true
 			}
 		}

core/headerchain.go

@@ -104,6 +104,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
 		}
 	}
 	hc.currentHeaderHash = hc.CurrentHeader().Hash()
+	headHeaderGauge.Update(hc.CurrentHeader().Number.Int64())
 
 	return hc, nil
 }
@@ -185,12 +186,12 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
 
 		hc.currentHeaderHash = hash
 		hc.currentHeader.Store(types.CopyHeader(header))
+		headHeaderGauge.Update(header.Number.Int64())
 
 		status = CanonStatTy
 	} else {
 		status = SideStatTy
 	}
-
 	hc.headerCache.Add(hash, header)
 	hc.numberCache.Add(hash, number)
 
@@ -456,6 +457,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
 
 	hc.currentHeader.Store(head)
 	hc.currentHeaderHash = head.Hash()
+	headHeaderGauge.Update(head.Number.Int64())
 }
 
 type (
@@ -508,6 +510,7 @@ func (hc *HeaderChain) SetHead(head uint64, updateFn UpdateHeadBlocksCallback, d
 
 		hc.currentHeader.Store(parent)
 		hc.currentHeaderHash = parentHash
+		headHeaderGauge.Update(parent.Number.Int64())
 	}
 	batch.Write()
 

core/rawdb/freezer.go

@@ -80,8 +80,9 @@ type freezer struct {
 func newFreezer(datadir string, namespace string) (*freezer, error) {
 	// Create the initial freezer object
 	var (
-		readMeter  = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
+		readMeter   = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
-		writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
+		writeMeter  = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
+		sizeCounter = metrics.NewRegisteredCounter(namespace+"ancient/size", nil)
 	)
 	// Ensure the datadir is not a symbolic link if it exists.
 	if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
@@ -102,7 +103,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
 		instanceLock: lock,
 	}
 	for name, disableSnappy := range freezerNoSnappy {
-		table, err := newTable(datadir, name, readMeter, writeMeter, disableSnappy)
+		table, err := newTable(datadir, name, readMeter, writeMeter, sizeCounter, disableSnappy)
 		if err != nil {
 			for _, table := range freezer.tables {
 				table.Close()

core/rawdb/freezer_table.go

@@ -94,17 +94,18 @@ type freezerTable struct {
 	// to count how many historic items have gone missing.
 	itemOffset uint32 // Offset (number of discarded items)
 
-	headBytes  uint32        // Number of bytes written to the head file
+	headBytes   uint32          // Number of bytes written to the head file
-	readMeter  metrics.Meter // Meter for measuring the effective amount of data read
+	readMeter   metrics.Meter   // Meter for measuring the effective amount of data read
-	writeMeter metrics.Meter // Meter for measuring the effective amount of data written
+	writeMeter  metrics.Meter   // Meter for measuring the effective amount of data written
+	sizeCounter metrics.Counter // Counter for tracking the combined size of all freezer tables
 
 	logger log.Logger   // Logger with database path and table name ambedded
 	lock   sync.RWMutex // Mutex protecting the data file descriptors
 }
 
 // newTable opens a freezer table with default settings - 2G files
-func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, disableSnappy bool) (*freezerTable, error) {
+func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, disableSnappy bool) (*freezerTable, error) {
-	return newCustomTable(path, name, readMeter, writeMeter, 2*1000*1000*1000, disableSnappy)
+	return newCustomTable(path, name, readMeter, writeMeter, sizeCounter, 2*1000*1000*1000, disableSnappy)
 }
 
 // openFreezerFileForAppend opens a freezer table file and seeks to the end
@@ -148,7 +149,7 @@ func truncateFreezerFile(file *os.File, size int64) error {
 // newCustomTable opens a freezer table, creating the data and index files if they are
 // non existent. Both files are truncated to the shortest common length to ensure
 // they don't go out of sync.
-func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
+func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
 	// Ensure the containing directory exists and open the indexEntry file
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return nil, err
@@ -171,6 +172,7 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete
 		files:         make(map[uint32]*os.File),
 		readMeter:     readMeter,
 		writeMeter:    writeMeter,
+		sizeCounter:   sizeCounter,
 		name:          name,
 		path:          path,
 		logger:        log.New("database", path, "table", name),
@@ -181,6 +183,14 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete
 		tab.Close()
 		return nil, err
 	}
+	// Initialize the starting size counter
+	size, err := tab.sizeNolock()
+	if err != nil {
+		tab.Close()
+		return nil, err
+	}
+	tab.sizeCounter.Inc(int64(size))
+
 	return tab, nil
 }
 
@@ -321,6 +331,11 @@ func (t *freezerTable) truncate(items uint64) error {
 	if atomic.LoadUint64(&t.items) <= items {
 		return nil
 	}
+	// We need to truncate, save the old size for metrics tracking
+	oldSize, err := t.sizeNolock()
+	if err != nil {
+		return err
+	}
 	// Something's out of sync, truncate the table's offset index
 	t.logger.Warn("Truncating freezer table", "items", t.items, "limit", items)
 	if err := truncateFreezerFile(t.index, int64(items+1)*indexEntrySize); err != nil {
@@ -355,6 +370,14 @@ func (t *freezerTable) truncate(items uint64) error {
 	// All data files truncated, set internal counters and return
 	atomic.StoreUint64(&t.items, items)
 	atomic.StoreUint32(&t.headBytes, expected.offset)
+
+	// Retrieve the new size and update the total size counter
+	newSize, err := t.sizeNolock()
+	if err != nil {
+		return err
+	}
+	t.sizeCounter.Dec(int64(oldSize - newSize))
+
 	return nil
 }
 
@@ -483,7 +506,10 @@ func (t *freezerTable) Append(item uint64, blob []byte) error {
 	}
 	// Write indexEntry
 	t.index.Write(idx.marshallBinary())
+
 	t.writeMeter.Mark(int64(bLen + indexEntrySize))
+	t.sizeCounter.Inc(int64(bLen + indexEntrySize))
+
 	atomic.AddUint64(&t.items, 1)
 	return nil
 }
@@ -562,6 +588,12 @@ func (t *freezerTable) size() (uint64, error) {
 	t.lock.RLock()
 	defer t.lock.RUnlock()
 
+	return t.sizeNolock()
+}
+
+// sizeNolock returns the total data size in the freezer table without obtaining
+// the mutex first.
+func (t *freezerTable) sizeNolock() (uint64, error) {
 	stat, err := t.index.Stat()
 	if err != nil {
 		return 0, err

core/rawdb/freezer_table_test.go

@@ -56,7 +56,7 @@ func TestFreezerBasics(t *testing.T) {
 	// set cutoff at 50 bytes
 	f, err := newCustomTable(os.TempDir(),
 		fmt.Sprintf("unittest-%d", rand.Uint64()),
-		metrics.NewMeter(), metrics.NewMeter(), 50, true)
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter(), 50, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -98,12 +98,12 @@ func TestFreezerBasicsClosing(t *testing.T) {
 	t.Parallel()
 	// set cutoff at 50 bytes
 	var (
-		fname  = fmt.Sprintf("basics-close-%d", rand.Uint64())
+		fname      = fmt.Sprintf("basics-close-%d", rand.Uint64())
-		m1, m2 = metrics.NewMeter(), metrics.NewMeter()
+		rm, wm, sc = metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
-		f      *freezerTable
+		f          *freezerTable
-		err    error
+		err        error
 	)
-	f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+	f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +112,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
 		data := getChunk(15, x)
 		f.Append(uint64(x), data)
 		f.Close()
-		f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+		f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 	}
 	defer f.Close()
 
@@ -126,7 +126,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
 			t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
 		}
 		f.Close()
-		f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+		f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -136,11 +136,11 @@ func TestFreezerBasicsClosing(t *testing.T) {
 // TestFreezerRepairDanglingHead tests that we can recover if index entries are removed
 func TestFreezerRepairDanglingHead(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
 
 	{ // Fill table
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -169,7 +169,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
 	idxFile.Close()
 	// Now open it again
 	{
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		// The last item should be missing
 		if _, err = f.Retrieve(0xff); err == nil {
 			t.Errorf("Expected error for missing index entry")
@@ -184,11 +184,11 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
 // TestFreezerRepairDanglingHeadLarge tests that we can recover if very many index entries are removed
 func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
 
 	{ // Fill a table and close it
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -216,7 +216,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 	idxFile.Close()
 	// Now open it again
 	{
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		// The first item should be there
 		if _, err = f.Retrieve(0); err != nil {
 			t.Fatal(err)
@@ -234,7 +234,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 	}
 	// And if we open it, we should now be able to read all of them (new values)
 	{
-		f, _ := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		for y := 1; y < 255; y++ {
 			exp := getChunk(15, ^y)
 			got, err := f.Retrieve(uint64(y))
@@ -251,11 +251,11 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 // TestSnappyDetection tests that we fail to open a snappy database and vice versa
 func TestSnappyDetection(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("snappytest-%d", rand.Uint64())
 	// Open with snappy
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -268,7 +268,7 @@ func TestSnappyDetection(t *testing.T) {
 	}
 	// Open without snappy
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, false)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, false)
 		if _, err = f.Retrieve(0); err == nil {
 			f.Close()
 			t.Fatalf("expected empty table")
@@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
 
 	// Open with snappy
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		// There should be 255 items
 		if _, err = f.Retrieve(0xfe); err != nil {
 			f.Close()
@@ -302,11 +302,11 @@ func assertFileSize(f string, size int64) error {
 // the index is repaired
 func TestFreezerRepairDanglingIndex(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64())
 
 	{ // Fill a table and close it
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -342,7 +342,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
 	// 45, 45, 15
 	// with 3+3+1 items
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -359,11 +359,11 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
 func TestFreezerTruncate(t *testing.T) {
 
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("truncation-%d", rand.Uint64())
 
 	{ // Fill table
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -380,7 +380,7 @@ func TestFreezerTruncate(t *testing.T) {
 	}
 	// Reopen, truncate
 	{
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -402,10 +402,10 @@ func TestFreezerTruncate(t *testing.T) {
 // That will rewind the index, and _should_ truncate the head file
 func TestFreezerRepairFirstFile(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64())
 	{ // Fill table
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -433,7 +433,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
 	}
 	// Reopen
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -458,10 +458,10 @@ func TestFreezerRepairFirstFile(t *testing.T) {
 // - check that we did not keep the rdonly file descriptors
 func TestFreezerReadAndTruncate(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("read_truncate-%d", rand.Uint64())
 	{ // Fill table
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -478,7 +478,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
 	}
 	// Reopen and read all files
 	{
-		f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -504,10 +504,10 @@ func TestFreezerReadAndTruncate(t *testing.T) {
 
 func TestOffset(t *testing.T) {
 	t.Parallel()
-	wm, rm := metrics.NewMeter(), metrics.NewMeter()
+	rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
 	fname := fmt.Sprintf("offset-%d", rand.Uint64())
 	{ // Fill table
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 40, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -563,7 +563,7 @@ func TestOffset(t *testing.T) {
 	}
 	// Now open again
 	{
-		f, err := newCustomTable(os.TempDir(), fname, rm, wm, 40, true)
+		f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true)
 		if err != nil {
 			t.Fatal(err)
 		}

core/tx_list.go

@@ -418,9 +418,9 @@ func (l *txPricedList) Put(tx *types.Transaction) {
 // Removed notifies the prices transaction list that an old transaction dropped
 // from the pool. The list will just keep a counter of stale objects and update
 // the heap if a large enough ratio of transactions go stale.
-func (l *txPricedList) Removed() {
+func (l *txPricedList) Removed(count int) {
 	// Bump the stale counter, but exit if still too low (< 25%)
-	l.stales++
+	l.stales += count
 	if l.stales <= len(*l.items)/4 {
 		return
 	}

core/tx_pool.go

@@ -85,20 +85,25 @@ var (
 
 var (
 	// Metrics for the pending pool
-	pendingDiscardCounter   = metrics.NewRegisteredCounter("txpool/pending/discard", nil)
+	pendingDiscardMeter   = metrics.NewRegisteredMeter("txpool/pending/discard", nil)
-	pendingReplaceCounter   = metrics.NewRegisteredCounter("txpool/pending/replace", nil)
+	pendingReplaceMeter   = metrics.NewRegisteredMeter("txpool/pending/replace", nil)
-	pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting
+	pendingRateLimitMeter = metrics.NewRegisteredMeter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting
-	pendingNofundsCounter   = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil)   // Dropped due to out-of-funds
+	pendingNofundsMeter   = metrics.NewRegisteredMeter("txpool/pending/nofunds", nil)   // Dropped due to out-of-funds
 
 	// Metrics for the queued pool
-	queuedDiscardCounter   = metrics.NewRegisteredCounter("txpool/queued/discard", nil)
+	queuedDiscardMeter   = metrics.NewRegisteredMeter("txpool/queued/discard", nil)
-	queuedReplaceCounter   = metrics.NewRegisteredCounter("txpool/queued/replace", nil)
+	queuedReplaceMeter   = metrics.NewRegisteredMeter("txpool/queued/replace", nil)
-	queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting
+	queuedRateLimitMeter = metrics.NewRegisteredMeter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting
-	queuedNofundsCounter   = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil)   // Dropped due to out-of-funds
+	queuedNofundsMeter   = metrics.NewRegisteredMeter("txpool/queued/nofunds", nil)   // Dropped due to out-of-funds
 
 	// General tx metrics
-	invalidTxCounter     = metrics.NewRegisteredCounter("txpool/invalid", nil)
+	validMeter         = metrics.NewRegisteredMeter("txpool/valid", nil)
-	underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil)
+	invalidTxMeter     = metrics.NewRegisteredMeter("txpool/invalid", nil)
+	underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil)
+
+	pendingCounter = metrics.NewRegisteredCounter("txpool/pending", nil)
+	queuedCounter  = metrics.NewRegisteredCounter("txpool/queued", nil)
+	localCounter   = metrics.NewRegisteredCounter("txpool/local", nil)
 )
 
 // TxStatus is the current status of a transaction as seen by the pool.
@@ -661,7 +666,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 	// If the transaction fails basic validation, discard it
 	if err := pool.validateTx(tx, local); err != nil {
 		log.Trace("Discarding invalid transaction", "hash", hash, "err", err)
-		invalidTxCounter.Inc(1)
+		invalidTxMeter.Mark(1)
 		return false, err
 	}
 	// If the transaction pool is full, discard underpriced transactions
@@ -669,14 +674,14 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		// If the new transaction is underpriced, don't accept it
 		if !local && pool.priced.Underpriced(tx, pool.locals) {
 			log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice())
-			underpricedTxCounter.Inc(1)
+			underpricedTxMeter.Mark(1)
 			return false, ErrUnderpriced
 		}
 		// New transaction is better than our worse ones, make room for it
 		drop := pool.priced.Discard(pool.all.Count()-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals)
 		for _, tx := range drop {
 			log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice())
-			underpricedTxCounter.Inc(1)
+			underpricedTxMeter.Mark(1)
 			pool.removeTx(tx.Hash(), false)
 		}
 	}
@@ -686,14 +691,14 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 		// Nonce already pending, check if required price bump is met
 		inserted, old := list.Add(tx, pool.config.PriceBump)
 		if !inserted {
-			pendingDiscardCounter.Inc(1)
+			pendingDiscardMeter.Mark(1)
 			return false, ErrReplaceUnderpriced
 		}
 		// New transaction is better, replace old one
 		if old != nil {
 			pool.all.Remove(old.Hash())
-			pool.priced.Removed()
+			pool.priced.Removed(1)
-			pendingReplaceCounter.Inc(1)
+			pendingReplaceMeter.Mark(1)
 		}
 		pool.all.Add(tx)
 		pool.priced.Put(tx)
@@ -718,6 +723,9 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) {
 			pool.locals.add(from)
 		}
 	}
+	if local || pool.locals.contains(from) {
+		localCounter.Inc(1)
+	}
 	pool.journalTx(from, tx)
 
 	log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To())
@@ -736,14 +744,17 @@ func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, er
 	inserted, old := pool.queue[from].Add(tx, pool.config.PriceBump)
 	if !inserted {
 		// An older transaction was better, discard this
-		queuedDiscardCounter.Inc(1)
+		queuedDiscardMeter.Mark(1)
 		return false, ErrReplaceUnderpriced
 	}
 	// Discard any previous transaction and mark this
 	if old != nil {
 		pool.all.Remove(old.Hash())
-		pool.priced.Removed()
+		pool.priced.Removed(1)
-		queuedReplaceCounter.Inc(1)
+		queuedReplaceMeter.Mark(1)
+	} else {
+		// Nothing was replaced, bump the queued counter
+		queuedCounter.Inc(1)
 	}
 	if pool.all.Get(hash) == nil {
 		pool.all.Add(tx)
@@ -779,17 +790,20 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T
 	if !inserted {
 		// An older transaction was better, discard this
 		pool.all.Remove(hash)
-		pool.priced.Removed()
+		pool.priced.Removed(1)
 
-		pendingDiscardCounter.Inc(1)
+		pendingDiscardMeter.Mark(1)
 		return false
 	}
 	// Otherwise discard any previous transaction and mark this
 	if old != nil {
 		pool.all.Remove(old.Hash())
-		pool.priced.Removed()
+		pool.priced.Removed(1)
 
-		pendingReplaceCounter.Inc(1)
+		pendingReplaceMeter.Mark(1)
+	} else {
+		// Nothing was replaced, bump the pending counter
+		pendingCounter.Inc(1)
 	}
 	// Failsafe to work around direct pending inserts (tests)
 	if pool.all.Get(hash) == nil {
@@ -844,6 +858,8 @@ func (pool *TxPool) addTx(tx *types.Transaction, local bool) error {
 	if err != nil {
 		return err
 	}
+	validMeter.Mark(1)
+
 	// If we added a new transaction, run promotion checks and return
 	if !replace {
 		from, _ := types.Sender(pool.signer, tx) // already validated
@@ -878,6 +894,8 @@ func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
 			dirty[from] = struct{}{}
 		}
 	}
+	validMeter.Mark(int64(len(dirty)))
+
 	// Only reprocess the internal state if something was actually added
 	if len(dirty) > 0 {
 		addrs := make([]common.Address, 0, len(dirty))
@@ -928,7 +946,10 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 	// Remove it from the list of known transactions
 	pool.all.Remove(hash)
 	if outofbound {
-		pool.priced.Removed()
+		pool.priced.Removed(1)
+	}
+	if pool.locals.contains(addr) {
+		localCounter.Dec(1)
 	}
 	// Remove the transaction from the pending lists and reset the account nonce
 	if pending := pool.pending[addr]; pending != nil {
@@ -946,12 +967,17 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 			if nonce := tx.Nonce(); pool.pendingState.GetNonce(addr) > nonce {
 				pool.pendingState.SetNonce(addr, nonce)
 			}
+			// Reduce the pending counter
+			pendingCounter.Dec(int64(1 + len(invalids)))
 			return
 		}
 	}
 	// Transaction is in the future queue
 	if future := pool.queue[addr]; future != nil {
-		future.Remove(tx)
+		if removed, _ := future.Remove(tx); removed {
+			// Reduce the queued counter
+			queuedCounter.Dec(1)
+		}
 		if future.Empty() {
 			delete(pool.queue, addr)
 		}
@@ -979,38 +1005,48 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			continue // Just in case someone calls with a non existing account
 		}
 		// Drop all transactions that are deemed too old (low nonce)
-		for _, tx := range list.Forward(pool.currentState.GetNonce(addr)) {
+		forwards := list.Forward(pool.currentState.GetNonce(addr))
+		for _, tx := range forwards {
 			hash := tx.Hash()
-			log.Trace("Removed old queued transaction", "hash", hash)
 			pool.all.Remove(hash)
-			pool.priced.Removed()
+			log.Trace("Removed old queued transaction", "hash", hash)
 		}
 		// Drop all transactions that are too costly (low balance or out of gas)
 		drops, _ := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas)
 		for _, tx := range drops {
 			hash := tx.Hash()
-			log.Trace("Removed unpayable queued transaction", "hash", hash)
 			pool.all.Remove(hash)
-			pool.priced.Removed()
+			log.Trace("Removed unpayable queued transaction", "hash", hash)
-			queuedNofundsCounter.Inc(1)
 		}
+		queuedNofundsMeter.Mark(int64(len(drops)))
+
 		// Gather all executable transactions and promote them
-		for _, tx := range list.Ready(pool.pendingState.GetNonce(addr)) {
+		readies := list.Ready(pool.pendingState.GetNonce(addr))
+		for _, tx := range readies {
 			hash := tx.Hash()
 			if pool.promoteTx(addr, hash, tx) {
 				log.Trace("Promoting queued transaction", "hash", hash)
 				promoted = append(promoted, tx)
 			}
 		}
+		queuedCounter.Dec(int64(len(readies)))
+
 		// Drop all transactions over the allowed limit
+		var caps types.Transactions
 		if !pool.locals.contains(addr) {
-			for _, tx := range list.Cap(int(pool.config.AccountQueue)) {
+			caps = list.Cap(int(pool.config.AccountQueue))
+			for _, tx := range caps {
 				hash := tx.Hash()
 				pool.all.Remove(hash)
-				pool.priced.Removed()
-				queuedRateLimitCounter.Inc(1)
 				log.Trace("Removed cap-exceeding queued transaction", "hash", hash)
 			}
+			queuedRateLimitMeter.Mark(int64(len(caps)))
+		}
+		// Mark all the items dropped as removed
+		pool.priced.Removed(len(forwards) + len(drops) + len(caps))
+		queuedCounter.Dec(int64(len(forwards) + len(drops) + len(caps)))
+		if pool.locals.contains(addr) {
+			localCounter.Dec(int64(len(forwards) + len(drops) + len(caps)))
 		}
 		// Delete the entire queue entry if it became empty.
 		if list.Empty() {
@@ -1052,11 +1088,12 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 				for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold {
 					for i := 0; i < len(offenders)-1; i++ {
 						list := pool.pending[offenders[i]]
-						for _, tx := range list.Cap(list.Len() - 1) {
+
+						caps := list.Cap(list.Len() - 1)
+						for _, tx := range caps {
 							// Drop the transaction from the global pools too
 							hash := tx.Hash()
 							pool.all.Remove(hash)
-							pool.priced.Removed()
 
 							// Update the account nonce to the dropped transaction
 							if nonce := tx.Nonce(); pool.pendingState.GetNonce(offenders[i]) > nonce {
@@ -1064,6 +1101,11 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 							}
 							log.Trace("Removed fairness-exceeding pending transaction", "hash", hash)
 						}
+						pool.priced.Removed(len(caps))
+						pendingCounter.Dec(int64(len(caps)))
+						if pool.locals.contains(offenders[i]) {
+							localCounter.Dec(int64(len(caps)))
+						}
 						pending--
 					}
 				}
@@ -1074,11 +1116,12 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			for pending > pool.config.GlobalSlots && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > pool.config.AccountSlots {
 				for _, addr := range offenders {
 					list := pool.pending[addr]
-					for _, tx := range list.Cap(list.Len() - 1) {
+
+					caps := list.Cap(list.Len() - 1)
+					for _, tx := range caps {
 						// Drop the transaction from the global pools too
 						hash := tx.Hash()
 						pool.all.Remove(hash)
-						pool.priced.Removed()
 
 						// Update the account nonce to the dropped transaction
 						if nonce := tx.Nonce(); pool.pendingState.GetNonce(addr) > nonce {
@@ -1086,11 +1129,16 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 						}
 						log.Trace("Removed fairness-exceeding pending transaction", "hash", hash)
 					}
+					pool.priced.Removed(len(caps))
+					pendingCounter.Dec(int64(len(caps)))
+					if pool.locals.contains(addr) {
+						localCounter.Dec(int64(len(caps)))
+					}
 					pending--
 				}
 			}
 		}
-		pendingRateLimitCounter.Inc(int64(pendingBeforeCap - pending))
+		pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending))
 	}
 	// If we've queued more transactions than the hard limit, drop oldest ones
 	queued := uint64(0)
@@ -1120,7 +1168,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 					pool.removeTx(tx.Hash(), true)
 				}
 				drop -= size
-				queuedRateLimitCounter.Inc(int64(size))
+				queuedRateLimitMeter.Mark(int64(size))
 				continue
 			}
 			// Otherwise drop only last few transactions
@@ -1128,7 +1176,7 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) {
 			for i := len(txs) - 1; i >= 0 && drop > 0; i-- {
 				pool.removeTx(txs[i].Hash(), true)
 				drop--
-				queuedRateLimitCounter.Inc(1)
+				queuedRateLimitMeter.Mark(1)
 			}
 		}
 	}
@@ -1143,11 +1191,11 @@ func (pool *TxPool) demoteUnexecutables() {
 		nonce := pool.currentState.GetNonce(addr)
 
 		// Drop all transactions that are deemed too old (low nonce)
-		for _, tx := range list.Forward(nonce) {
+		olds := list.Forward(nonce)
+		for _, tx := range olds {
 			hash := tx.Hash()
-			log.Trace("Removed old pending transaction", "hash", hash)
 			pool.all.Remove(hash)
-			pool.priced.Removed()
+			log.Trace("Removed old pending transaction", "hash", hash)
 		}
 		// Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later
 		drops, invalids := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas)
@@ -1155,21 +1203,28 @@ func (pool *TxPool) demoteUnexecutables() {
 			hash := tx.Hash()
 			log.Trace("Removed unpayable pending transaction", "hash", hash)
 			pool.all.Remove(hash)
-			pool.priced.Removed()
-			pendingNofundsCounter.Inc(1)
 		}
+		pool.priced.Removed(len(olds) + len(drops))
+		pendingNofundsMeter.Mark(int64(len(drops)))
+
 		for _, tx := range invalids {
 			hash := tx.Hash()
 			log.Trace("Demoting pending transaction", "hash", hash)
 			pool.enqueueTx(hash, tx)
 		}
+		pendingCounter.Dec(int64(len(olds) + len(drops) + len(invalids)))
+		if pool.locals.contains(addr) {
+			localCounter.Dec(int64(len(olds) + len(drops) + len(invalids)))
+		}
 		// If there's a gap in front, alert (should never happen) and postpone all transactions
 		if list.Len() > 0 && list.txs.Get(nonce) == nil {
-			for _, tx := range list.Cap(0) {
+			gapped := list.Cap(0)
+			for _, tx := range gapped {
 				hash := tx.Hash()
 				log.Error("Demoting invalidated transaction", "hash", hash)
 				pool.enqueueTx(hash, tx)
 			}
+			pendingCounter.Inc(int64(len(gapped)))
 		}
 		// Delete the entire queue entry if it became empty.
 		if list.Empty() {

core/types/transaction.go

@@ -252,7 +252,9 @@ func (tx *Transaction) Cost() *big.Int {
 	return total
 }
 
-func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
+// RawSignatureValues returns the V, R, S signature values of the transaction.
+// The return values should not be modified by the caller.
+func (tx *Transaction) RawSignatureValues() (v, r, s *big.Int) {
 	return tx.data.V, tx.data.R, tx.data.S
 }
 

eth/backend.go

@@ -481,6 +481,7 @@ func (s *Ethereum) EthVersion() int                    { return int(s.protocolMa
 func (s *Ethereum) NetVersion() uint64                 { return s.networkID }
 func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManager.downloader }
 func (s *Ethereum) Synced() bool                       { return atomic.LoadUint32(&s.protocolManager.acceptTxs) == 1 }
+func (s *Ethereum) ArchiveMode() bool                  { return s.config.NoPruning }
 
 // Protocols implements node.Service, returning all the currently configured
 // network protocols to start.

eth/downloader/statesync.go

@@ -442,9 +442,7 @@ func (s *stateSync) process(req *stateReq) (int, error) {
 		default:
 			return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
 		}
-		if _, ok := req.tasks[hash]; ok {
+		delete(req.tasks, hash)
-			delete(req.tasks, hash)
-		}
 	}
 	// Put unfulfilled tasks back into the retry queue
 	npeers := s.d.peers.Len()

eth/peer.go

@@ -196,9 +196,13 @@ func (p *peer) MarkTransaction(hash common.Hash) {
 // SendTransactions sends transactions to the peer and includes the hashes
 // in its transaction hash set for future reference.
 func (p *peer) SendTransactions(txs types.Transactions) error {
+	// Mark all the transactions as known, but ensure we don't overflow our limits
 	for _, tx := range txs {
 		p.knownTxs.Add(tx.Hash())
 	}
+	for p.knownTxs.Cardinality() >= maxKnownTxs {
+		p.knownTxs.Pop()
+	}
 	return p2p.Send(p.rw, TxMsg, txs)
 }
 
@@ -207,9 +211,13 @@ func (p *peer) SendTransactions(txs types.Transactions) error {
 func (p *peer) AsyncSendTransactions(txs []*types.Transaction) {
 	select {
 	case p.queuedTxs <- txs:
+		// Mark all the transactions as known, but ensure we don't overflow our limits
 		for _, tx := range txs {
 			p.knownTxs.Add(tx.Hash())
 		}
+		for p.knownTxs.Cardinality() >= maxKnownTxs {
+			p.knownTxs.Pop()
+		}
 	default:
 		p.Log().Debug("Dropping transaction propagation", "count", len(txs))
 	}
@@ -218,9 +226,13 @@ func (p *peer) AsyncSendTransactions(txs []*types.Transaction) {
 // SendNewBlockHashes announces the availability of a number of blocks through
 // a hash notification.
 func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error {
+	// Mark all the block hashes as known, but ensure we don't overflow our limits
 	for _, hash := range hashes {
 		p.knownBlocks.Add(hash)
 	}
+	for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+		p.knownBlocks.Pop()
+	}
 	request := make(newBlockHashesData, len(hashes))
 	for i := 0; i < len(hashes); i++ {
 		request[i].Hash = hashes[i]
@@ -235,7 +247,11 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error
 func (p *peer) AsyncSendNewBlockHash(block *types.Block) {
 	select {
 	case p.queuedAnns <- block:
+		// Mark all the block hash as known, but ensure we don't overflow our limits
 		p.knownBlocks.Add(block.Hash())
+		for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+			p.knownBlocks.Pop()
+		}
 	default:
 		p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash())
 	}
@@ -243,7 +259,11 @@ func (p *peer) AsyncSendNewBlockHash(block *types.Block) {
 
 // SendNewBlock propagates an entire block to a remote peer.
 func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
+	// Mark all the block hash as known, but ensure we don't overflow our limits
 	p.knownBlocks.Add(block.Hash())
+	for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+		p.knownBlocks.Pop()
+	}
 	return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td})
 }
 
@@ -252,7 +272,11 @@ func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
 func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) {
 	select {
 	case p.queuedProps <- &propEvent{block: block, td: td}:
+		// Mark all the block hash as known, but ensure we don't overflow our limits
 		p.knownBlocks.Add(block.Hash())
+		for p.knownBlocks.Cardinality() >= maxKnownBlocks {
+			p.knownBlocks.Pop()
+		}
 	default:
 		p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash())
 	}

ethclient/ethclient.go

@@ -61,6 +61,16 @@ func (ec *Client) Close() {
 
 // Blockchain Access
 
+// ChainId retrieves the current chain ID for transaction replay protection.
+func (ec *Client) ChainID(ctx context.Context) (*big.Int, error) {
+	var result hexutil.Big
+	err := ec.c.CallContext(ctx, &result, "eth_chainId")
+	if err != nil {
+		return nil, err
+	}
+	return (*big.Int)(&result), err
+}
+
 // BlockByHash returns the given full block.
 //
 // Note that loading full blocks requires two requests. Use HeaderByHash

ethclient/ethclient_test.go

@@ -408,5 +408,20 @@ func TestBlockReceipts(t *testing.T) {
 
 	if receipts[0].BlockHash != blocks[1].Hash() {
 		t.Fatalf("BlockReceipts block hash mismatch, got %v, want %v", receipts[0].BlockHash, blocks[1].Hash())
+
+func TestChainID(t *testing.T) {
+	backend, _ := newTestBackend(t, false)
+	client, _ := backend.Attach()
+	defer backend.Stop()
+	defer client.Close()
+	ec := NewClient(client)
+
+	id, err := ec.ChainID(context.Background())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if id == nil || id.Cmp(params.AllEthashProtocolChanges.ChainID) != 0 {
+		t.Fatalf("ChainID returned wrong number: %+v", id)
+
 	}
 }

ethdb/leveldb/leveldb.go

@@ -67,6 +67,7 @@ type Database struct {
 	compWriteMeter   metrics.Meter // Meter for measuring the data written during compaction
 	writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction
 	writeDelayMeter  metrics.Meter // Meter for measuring the write delay duration due to database compaction
+	diskSizeGauge    metrics.Gauge // Gauge for tracking the size of all the levels in the database
 	diskReadMeter    metrics.Meter // Meter for measuring the effective amount of data read
 	diskWriteMeter   metrics.Meter // Meter for measuring the effective amount of data written
 
@@ -112,6 +113,7 @@ func New(file string, cache int, handles int, namespace string) (*Database, erro
 	ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil)
 	ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil)
 	ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil)
+	ldb.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil)
 	ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil)
 	ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil)
 	ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil)
@@ -233,7 +235,7 @@ func (db *Database) meter(refresh time.Duration) {
 	// Create the counters to store current and previous compaction values
 	compactions := make([][]float64, 2)
 	for i := 0; i < 2; i++ {
-		compactions[i] = make([]float64, 3)
+		compactions[i] = make([]float64, 4)
 	}
 	// Create storage for iostats.
 	var iostats [2]float64
@@ -279,7 +281,7 @@ func (db *Database) meter(refresh time.Duration) {
 			if len(parts) != 6 {
 				break
 			}
-			for idx, counter := range parts[3:] {
+			for idx, counter := range parts[2:] {
 				value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64)
 				if err != nil {
 					db.log.Error("Compaction entry parsing failed", "err", err)
@@ -290,16 +292,18 @@ func (db *Database) meter(refresh time.Duration) {
 			}
 		}
 		// Update all the requested meters
+		if db.diskSizeGauge != nil {
+			db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024))
+		}
 		if db.compTimeMeter != nil {
-			db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000))
+			db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000))
 		}
 		if db.compReadMeter != nil {
-			db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024))
+			db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024))
 		}
 		if db.compWriteMeter != nil {
-			db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024))
+			db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024))
 		}
-
 		// Retrieve the write delay statistic
 		writedelay, err := db.db.GetProperty("leveldb.writedelay")
 		if err != nil {

internal/ethapi/api.go

@@ -529,6 +529,11 @@ func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI {
 	return &PublicBlockChainAPI{b}
 }
 
+// ChainId returns the chainID value for transaction replay protection.
+func (s *PublicBlockChainAPI) ChainId() *hexutil.Big {
+	return (*hexutil.Big)(s.b.ChainConfig().ChainID)
+}
+
 // BlockNumber returns the block number of the chain head.
 func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 {
 	header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available

les/handler.go

@@ -19,9 +19,11 @@ package les
 import (
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/big"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -44,6 +46,8 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
+var errTooManyInvalidRequest = errors.New("too many invalid requests made")
+
 const (
 	softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data.
 	estHeaderRlpSize  = 500             // Approximate size of an RLP encoded block header
@@ -524,6 +528,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number)
 					}
 					if origin == nil {
+						atomic.AddUint32(&p.invalidCount, 1)
 						break
 					}
 					headers = append(headers, origin)
@@ -570,7 +575,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						} else {
 							unknown = true
 						}
-
 					case !query.Reverse:
 						// Number based traversal towards the leaf block
 						query.Origin.Number += query.Skip + 1
@@ -628,15 +632,18 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						sendResponse(req.ReqID, 0, nil, task.servingTime)
 						return
 					}
+					// Retrieve the requested block body, stopping if enough was found
 					if bytes >= softResponseLimit {
 						break
 					}
-					// Retrieve the requested block body, stopping if enough was found
+					number := rawdb.ReadHeaderNumber(pm.chainDb, hash)
-					if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil {
+					if number == nil {
-						if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 {
+						atomic.AddUint32(&p.invalidCount, 1)
-							bodies = append(bodies, data)
+						continue
-							bytes += len(data)
+					}
-						}
+					if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 {
+						bodies = append(bodies, data)
+						bytes += len(data)
 					}
 				}
 				sendResponse(req.ReqID, uint64(reqCnt), p.ReplyBlockBodiesRLP(req.ReqID, bodies), task.done())
@@ -691,6 +698,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 					number := rawdb.ReadHeaderNumber(pm.chainDb, request.BHash)
 					if number == nil {
 						p.Log().Warn("Failed to retrieve block num for code", "hash", request.BHash)
+						atomic.AddUint32(&p.invalidCount, 1)
 						continue
 					}
 					header := rawdb.ReadHeader(pm.chainDb, request.BHash, *number)
@@ -698,11 +706,20 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						p.Log().Warn("Failed to retrieve header for code", "block", *number, "hash", request.BHash)
 						continue
 					}
+					// Refuse to search stale state data in the database since looking for
+					// a non-exist key is kind of expensive.
+					local := pm.blockchain.CurrentHeader().Number.Uint64()
+					if !pm.server.archiveMode && header.Number.Uint64()+core.TriesInMemory <= local {
+						p.Log().Debug("Reject stale code request", "number", header.Number.Uint64(), "head", local)
+						atomic.AddUint32(&p.invalidCount, 1)
+						continue
+					}
 					triedb := pm.blockchain.StateCache().TrieDB()
 
 					account, err := pm.getAccount(triedb, header.Root, common.BytesToHash(request.AccKey))
 					if err != nil {
 						p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err)
+						atomic.AddUint32(&p.invalidCount, 1)
 						continue
 					}
 					code, err := triedb.Node(common.BytesToHash(account.CodeHash))
@@ -769,9 +786,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 					}
 					// Retrieve the requested block's receipts, skipping if unknown to us
 					var results types.Receipts
-					if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil {
+					number := rawdb.ReadHeaderNumber(pm.chainDb, hash)
-						results = rawdb.ReadRawReceipts(pm.chainDb, hash, *number)
+					if number == nil {
+						atomic.AddUint32(&p.invalidCount, 1)
+						continue
 					}
+					results = rawdb.ReadRawReceipts(pm.chainDb, hash, *number)
 					if results == nil {
 						if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
 							continue
@@ -846,14 +866,28 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 
 						if number = rawdb.ReadHeaderNumber(pm.chainDb, request.BHash); number == nil {
 							p.Log().Warn("Failed to retrieve block num for proof", "hash", request.BHash)
+							atomic.AddUint32(&p.invalidCount, 1)
 							continue
 						}
 						if header = rawdb.ReadHeader(pm.chainDb, request.BHash, *number); header == nil {
 							p.Log().Warn("Failed to retrieve header for proof", "block", *number, "hash", request.BHash)
 							continue
 						}
+						// Refuse to search stale state data in the database since looking for
+						// a non-exist key is kind of expensive.
+						local := pm.blockchain.CurrentHeader().Number.Uint64()
+						if !pm.server.archiveMode && header.Number.Uint64()+core.TriesInMemory <= local {
+							p.Log().Debug("Reject stale trie request", "number", header.Number.Uint64(), "head", local)
+							atomic.AddUint32(&p.invalidCount, 1)
+							continue
+						}
 						root = header.Root
 					}
+					// If a header lookup failed (non existent), ignore subsequent requests for the same header
+					if root == (common.Hash{}) {
+						atomic.AddUint32(&p.invalidCount, 1)
+						continue
+					}
 					// Open the account or storage trie for the request
 					statedb := pm.blockchain.StateCache()
 
@@ -870,6 +904,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 						account, err := pm.getAccount(statedb.TrieDB(), root, common.BytesToHash(request.AccKey))
 						if err != nil {
 							p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err)
+							atomic.AddUint32(&p.invalidCount, 1)
 							continue
 						}
 						trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root)
@@ -1116,6 +1151,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			}
 		}
 	}
+	// If the client has made too much invalid request(e.g. request a non-exist data),
+	// reject them to prevent SPAM attack.
+	if atomic.LoadUint32(&p.invalidCount) > maxRequestErrors {
+		return errTooManyInvalidRequest
+	}
 	return nil
 }
 

les/handler_test.go

@@ -278,6 +278,31 @@ func testGetCode(t *testing.T, protocol int) {
 	}
 }
 
+// Tests that the stale contract codes can't be retrieved based on account addresses.
+func TestGetStaleCodeLes2(t *testing.T) { testGetStaleCode(t, 2) }
+func TestGetStaleCodeLes3(t *testing.T) { testGetStaleCode(t, 3) }
+
+func testGetStaleCode(t *testing.T, protocol int) {
+	server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil)
+	defer tearDown()
+	bc := server.pm.blockchain.(*core.BlockChain)
+
+	check := func(number uint64, expected [][]byte) {
+		req := &CodeReq{
+			BHash:  bc.GetHeaderByNumber(number).Hash(),
+			AccKey: crypto.Keccak256(testContractAddr[:]),
+		}
+		cost := server.tPeer.GetRequestCost(GetCodeMsg, 1)
+		sendRequest(server.tPeer.app, GetCodeMsg, 42, cost, []*CodeReq{req})
+		if err := expectResponse(server.tPeer.app, CodeMsg, 42, testBufLimit, expected); err != nil {
+			t.Errorf("codes mismatch: %v", err)
+		}
+	}
+	check(0, [][]byte{})                                                          // Non-exist contract
+	check(testContractDeployed, [][]byte{})                                       // Stale contract
+	check(bc.CurrentHeader().Number.Uint64(), [][]byte{testContractCodeDeployed}) // Fresh contract
+}
+
 // Tests that the transaction receipts can be retrieved based on hashes.
 func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) }
 
@@ -338,6 +363,43 @@ func testGetProofs(t *testing.T, protocol int) {
 	}
 }
 
+// Tests that the stale contract codes can't be retrieved based on account addresses.
+func TestGetStaleProofLes2(t *testing.T) { testGetStaleProof(t, 2) }
+func TestGetStaleProofLes3(t *testing.T) { testGetStaleProof(t, 3) }
+
+func testGetStaleProof(t *testing.T, protocol int) {
+	server, tearDown := newServerEnv(t, core.TriesInMemory+4, protocol, nil)
+	defer tearDown()
+	bc := server.pm.blockchain.(*core.BlockChain)
+
+	check := func(number uint64, wantOK bool) {
+		var (
+			header  = bc.GetHeaderByNumber(number)
+			account = crypto.Keccak256(testBankAddress.Bytes())
+		)
+		req := &ProofReq{
+			BHash: header.Hash(),
+			Key:   account,
+		}
+		cost := server.tPeer.GetRequestCost(GetProofsV2Msg, 1)
+		sendRequest(server.tPeer.app, GetProofsV2Msg, 42, cost, []*ProofReq{req})
+
+		var expected []rlp.RawValue
+		if wantOK {
+			proofsV2 := light.NewNodeSet()
+			t, _ := trie.New(header.Root, trie.NewDatabase(server.db))
+			t.Prove(crypto.Keccak256(account), 0, proofsV2)
+			expected = proofsV2.NodeList()
+		}
+		if err := expectResponse(server.tPeer.app, ProofsV2Msg, 42, testBufLimit, expected); err != nil {
+			t.Errorf("codes mismatch: %v", err)
+		}
+	}
+	check(0, false)                                 // Non-exist proof
+	check(2, false)                                 // Stale proof
+	check(bc.CurrentHeader().Number.Uint64(), true) // Fresh proof
+}
+
 // Tests that CHT proofs can be correctly retrieved.
 func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) }
 
@@ -535,9 +597,10 @@ func TestStopResumeLes3(t *testing.T) {
 	expBuf := testBufLimit
 	var reqID uint64
 
+	header := pm.blockchain.CurrentHeader()
 	req := func() {
 		reqID++
-		sendRequest(peer.app, GetBlockHeadersMsg, reqID, testCost, &getBlockHeadersData{Origin: hashOrNumber{Hash: common.Hash{1}}, Amount: 1})
+		sendRequest(peer.app, GetBlockHeadersMsg, reqID, testCost, &getBlockHeadersData{Origin: hashOrNumber{Hash: header.Hash()}, Amount: 1})
 	}
 
 	for i := 1; i <= 5; i++ {
@@ -545,8 +608,8 @@ func TestStopResumeLes3(t *testing.T) {
 		for expBuf >= testCost {
 			req()
 			expBuf -= testCost
-			if err := expectResponse(peer.app, BlockHeadersMsg, reqID, expBuf, nil); err != nil {
+			if err := expectResponse(peer.app, BlockHeadersMsg, reqID, expBuf, []*types.Header{header}); err != nil {
-				t.Errorf("expected response and failed: %v", err)
+				t.Fatalf("expected response and failed: %v", err)
 			}
 		}
 		// send some more requests in excess and expect a single StopMsg

les/peer.go

@@ -42,7 +42,10 @@ var (
 	errNotRegistered     = errors.New("peer is not registered")
 )
 
-const maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam)
+const (
+	maxRequestErrors  = 20 // number of invalid requests tolerated (makes the protocol less brittle but still avoids spam)
+	maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam)
+)
 
 // capacity limitation for parameter updates
 const (
@@ -69,7 +72,6 @@ const (
 
 type peer struct {
 	*p2p.Peer
-
 	rw p2p.MsgReadWriter
 
 	version int    // Protocol version negotiated
@@ -89,6 +91,7 @@ type peer struct {
 	// RequestProcessed is called
 	responseLock  sync.Mutex
 	responseCount uint64
+	invalidCount  uint32
 
 	poolEntry      *poolEntry
 	hasBlock       func(common.Hash, uint64, bool) bool
@@ -551,7 +554,14 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis
 			send = send.add("serveHeaders", nil)
 			send = send.add("serveChainSince", uint64(0))
 			send = send.add("serveStateSince", uint64(0))
-			send = send.add("serveRecentState", uint64(core.TriesInMemory-4))
+
+			// If local ethereum node is running in archive mode, advertise ourselves we have
+			// all version state data. Otherwise only recent state is available.
+			stateRecent := uint64(core.TriesInMemory - 4)
+			if server.archiveMode {
+				stateRecent = 0
+			}
+			send = send.add("serveRecentState", stateRecent)
 			send = send.add("txRelay", nil)
 		}
 		send = send.add("flowControl/BL", server.defParams.BufLimit)

les/server.go

@@ -51,6 +51,8 @@ const (
 type LesServer struct {
 	lesCommons
 
+	archiveMode bool // Flag whether the ethereum node runs in archive mode.
+
 	fcManager    *flowcontrol.ClientManager // nil if our node is client only
 	costTracker  *costTracker
 	testCost     uint64
@@ -93,7 +95,8 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) {
 		nil,
 		quitSync,
 		new(sync.WaitGroup),
-		config.ULC, eth.Synced)
+		config.ULC,
+		eth.Synced)
 	if err != nil {
 		return nil, err
 	}
@@ -120,6 +123,7 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) {
 			bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency),
 			protocolManager:  pm,
 		},
+		archiveMode:  eth.ArchiveMode(),
 		quitSync:     quitSync,
 		lesTopics:    lesTopics,
 		onlyAnnounce: config.OnlyAnnounce,

metrics/cpu.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package metrics
+
+import "github.com/elastic/gosigar"
+
+// CPUStats is the system and process CPU stats.
+type CPUStats struct {
+	GlobalTime int64 // Time spent by the CPU working on all processes
+	GlobalWait int64 // Time spent by waiting on disk for all processes
+	LocalTime  int64 // Time spent by the CPU working on this process
+}
+
+// ReadCPUStats retrieves the current CPU stats.
+func ReadCPUStats(stats *CPUStats) {
+	global := gosigar.Cpu{}
+	global.Get()
+
+	stats.GlobalTime = int64(global.User + global.Nice + global.Sys)
+	stats.GlobalWait = int64(global.Wait)
+	stats.LocalTime = getProcessCPUTime()
+}

metrics/cpu_syscall.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !windows
+
+package metrics
+
+import (
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// getProcessCPUTime retrieves the process' CPU time since program startup.
+func getProcessCPUTime() int64 {
+	var usage syscall.Rusage
+	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
+		log.Warn("Failed to retrieve CPU time", "err", err)
+		return 0
+	}
+	return int64(usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert
+}

metrics/cpu_windows.go

@@ -0,0 +1,23 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package metrics
+
+// getProcessCPUTime returns 0 on Windows as there is no system call to resolve
+// the actual process' CPU time.
+func getProcessCPUTime() int64 {
+	return 0
+}

metrics/metrics.go

@@ -61,18 +61,27 @@ func CollectProcessMetrics(refresh time.Duration) {
 	if !Enabled {
 		return
 	}
+	refreshFreq := int64(refresh / time.Second)
+
 	// Create the various data collectors
+	cpuStats := make([]*CPUStats, 2)
 	memstats := make([]*runtime.MemStats, 2)
 	diskstats := make([]*DiskStats, 2)
 	for i := 0; i < len(memstats); i++ {
+		cpuStats[i] = new(CPUStats)
 		memstats[i] = new(runtime.MemStats)
 		diskstats[i] = new(DiskStats)
 	}
 	// Define the various metrics to collect
+	cpuSysLoad := GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry)
+	cpuSysWait := GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry)
+	cpuProcLoad := GetOrRegisterGauge("system/cpu/procload", DefaultRegistry)
+
+	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
 	memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
 	memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
-	memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry)
+	memHeld := GetOrRegisterGauge("system/memory/held", DefaultRegistry)
-	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
+	memUsed := GetOrRegisterGauge("system/memory/used", DefaultRegistry)
 
 	var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
 	var diskReadBytesCounter, diskWriteBytesCounter Counter
@@ -91,11 +100,17 @@ func CollectProcessMetrics(refresh time.Duration) {
 		location1 := i % 2
 		location2 := (i - 1) % 2
 
+		ReadCPUStats(cpuStats[location1])
+		cpuSysLoad.Update((cpuStats[location1].GlobalTime - cpuStats[location2].GlobalTime) / refreshFreq)
+		cpuSysWait.Update((cpuStats[location1].GlobalWait - cpuStats[location2].GlobalWait) / refreshFreq)
+		cpuProcLoad.Update((cpuStats[location1].LocalTime - cpuStats[location2].LocalTime) / refreshFreq)
+
 		runtime.ReadMemStats(memstats[location1])
+		memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
 		memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs))
 		memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees))
-		memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc))
+		memHeld.Update(int64(memstats[location1].HeapSys - memstats[location1].HeapReleased))
-		memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
+		memUsed.Update(int64(memstats[location1].Alloc))
 
 		if ReadDiskStats(diskstats[location1]) == nil {
 			diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount)

p2p/dial.go

@@ -17,7 +17,6 @@
 package p2p
 
 import (
-	"container/heap"
 	"errors"
 	"fmt"
 	"net"
@@ -29,9 +28,10 @@ import (
 )
 
 const (
-	// This is the amount of time spent waiting in between
+	// This is the amount of time spent waiting in between redialing a certain node. The
-	// redialing a certain node.
+	// limit is a bit higher than inboundThrottleTime to prevent failing dials in small
-	dialHistoryExpiration = 30 * time.Second
+	// private networks.
+	dialHistoryExpiration = inboundThrottleTime + 5*time.Second
 
 	// Discovery lookups are throttled and can only run
 	// once every few seconds.
@@ -72,16 +72,16 @@ type dialstate struct {
 	ntab        discoverTable
 	netrestrict *netutil.Netlist
 	self        enode.ID
+	bootnodes   []*enode.Node // default dials when there are no peers
+	log         log.Logger
 
+	start         time.Time // time when the dialer was first used
 	lookupRunning bool
 	dialing       map[enode.ID]connFlag
 	lookupBuf     []*enode.Node // current discovery lookup results
 	randomNodes   []*enode.Node // filled from Table
 	static        map[enode.ID]*dialTask
-	hist          *dialHistory
+	hist          expHeap
-
-	start     time.Time     // time when the dialer was first used
-	bootnodes []*enode.Node // default dials when there are no peers
 }
 
 type discoverTable interface {
@@ -91,15 +91,6 @@ type discoverTable interface {
 	ReadRandomNodes([]*enode.Node) int
 }
 
-// the dial history remembers recent dials.
-type dialHistory []pastDial
-
-// pastDial is an entry in the dial history.
-type pastDial struct {
-	id  enode.ID
-	exp time.Time
-}
-
 type task interface {
 	Do(*Server)
 }
@@ -126,20 +117,23 @@ type waitExpireTask struct {
 	time.Duration
 }
 
-func newDialState(self enode.ID, static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate {
+func newDialState(self enode.ID, ntab discoverTable, maxdyn int, cfg *Config) *dialstate {
 	s := &dialstate{
 		maxDynDials: maxdyn,
 		ntab:        ntab,
 		self:        self,
-		netrestrict: netrestrict,
+		netrestrict: cfg.NetRestrict,
+		log:         cfg.Logger,
 		static:      make(map[enode.ID]*dialTask),
 		dialing:     make(map[enode.ID]connFlag),
-		bootnodes:   make([]*enode.Node, len(bootnodes)),
+		bootnodes:   make([]*enode.Node, len(cfg.BootstrapNodes)),
 		randomNodes: make([]*enode.Node, maxdyn/2),
-		hist:        new(dialHistory),
 	}
-	copy(s.bootnodes, bootnodes)
+	copy(s.bootnodes, cfg.BootstrapNodes)
-	for _, n := range static {
+	if s.log == nil {
+		s.log = log.Root()
+	}
+	for _, n := range cfg.StaticNodes {
 		s.addStatic(n)
 	}
 	return s
@@ -154,9 +148,6 @@ func (s *dialstate) addStatic(n *enode.Node) {
 func (s *dialstate) removeStatic(n *enode.Node) {
 	// This removes a task so future attempts to connect will not be made.
 	delete(s.static, n.ID())
-	// This removes a previous dial timestamp so that application
-	// can force a server to reconnect with chosen peer immediately.
-	s.hist.remove(n.ID())
 }
 
 func (s *dialstate) newTasks(nRunning int, peers map[enode.ID]*Peer, now time.Time) []task {
@@ -167,7 +158,7 @@ func (s *dialstate) newTasks(nRunning int, peers map[enode.ID]*Peer, now time.Ti
 	var newtasks []task
 	addDial := func(flag connFlag, n *enode.Node) bool {
 		if err := s.checkDial(n, peers); err != nil {
-			log.Trace("Skipping dial candidate", "id", n.ID(), "addr", &net.TCPAddr{IP: n.IP(), Port: n.TCP()}, "err", err)
+			s.log.Trace("Skipping dial candidate", "id", n.ID(), "addr", &net.TCPAddr{IP: n.IP(), Port: n.TCP()}, "err", err)
 			return false
 		}
 		s.dialing[n.ID()] = flag
@@ -196,7 +187,7 @@ func (s *dialstate) newTasks(nRunning int, peers map[enode.ID]*Peer, now time.Ti
 		err := s.checkDial(t.dest, peers)
 		switch err {
 		case errNotWhitelisted, errSelf:
-			log.Warn("Removing static dial candidate", "id", t.dest.ID, "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}, "err", err)
+			s.log.Warn("Removing static dial candidate", "id", t.dest.ID, "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}, "err", err)
 			delete(s.static, t.dest.ID())
 		case nil:
 			s.dialing[id] = t.flags
@@ -246,7 +237,7 @@ func (s *dialstate) newTasks(nRunning int, peers map[enode.ID]*Peer, now time.Ti
 	// This should prevent cases where the dialer logic is not ticked
 	// because there are no pending events.
 	if nRunning == 0 && len(newtasks) == 0 && s.hist.Len() > 0 {
-		t := &waitExpireTask{s.hist.min().exp.Sub(now)}
+		t := &waitExpireTask{s.hist.nextExpiry().Sub(now)}
 		newtasks = append(newtasks, t)
 	}
 	return newtasks
@@ -271,7 +262,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error {
 		return errSelf
 	case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()):
 		return errNotWhitelisted
-	case s.hist.contains(n.ID()):
+	case s.hist.contains(string(n.ID().Bytes())):
 		return errRecentlyDialed
 	}
 	return nil
@@ -280,7 +271,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error {
 func (s *dialstate) taskDone(t task, now time.Time) {
 	switch t := t.(type) {
 	case *dialTask:
-		s.hist.add(t.dest.ID(), now.Add(dialHistoryExpiration))
+		s.hist.add(string(t.dest.ID().Bytes()), now.Add(dialHistoryExpiration))
 		delete(s.dialing, t.dest.ID())
 	case *discoverTask:
 		s.lookupRunning = false
@@ -296,7 +287,7 @@ func (t *dialTask) Do(srv *Server) {
 	}
 	err := t.dial(srv, t.dest)
 	if err != nil {
-		log.Trace("Dial error", "task", t, "err", err)
+		srv.log.Trace("Dial error", "task", t, "err", err)
 		// Try resolving the ID of static nodes if dialing failed.
 		if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 {
 			if t.resolve(srv) {
@@ -314,7 +305,7 @@ func (t *dialTask) Do(srv *Server) {
 // The backoff delay resets when the node is found.
 func (t *dialTask) resolve(srv *Server) bool {
 	if srv.ntab == nil {
-		log.Debug("Can't resolve node", "id", t.dest.ID, "err", "discovery is disabled")
+		srv.log.Debug("Can't resolve node", "id", t.dest.ID, "err", "discovery is disabled")
 		return false
 	}
 	if t.resolveDelay == 0 {
@@ -330,13 +321,13 @@ func (t *dialTask) resolve(srv *Server) bool {
 		if t.resolveDelay > maxResolveDelay {
 			t.resolveDelay = maxResolveDelay
 		}
-		log.Debug("Resolving node failed", "id", t.dest.ID, "newdelay", t.resolveDelay)
+		srv.log.Debug("Resolving node failed", "id", t.dest.ID, "newdelay", t.resolveDelay)
 		return false
 	}
 	// The node was found.
 	t.resolveDelay = initialResolveDelay
 	t.dest = resolved
-	log.Debug("Resolved node", "id", t.dest.ID, "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()})
+	srv.log.Debug("Resolved node", "id", t.dest.ID, "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()})
 	return true
 }
 
@@ -385,49 +376,3 @@ func (t waitExpireTask) Do(*Server) {
 func (t waitExpireTask) String() string {
 	return fmt.Sprintf("wait for dial hist expire (%v)", t.Duration)
 }
-
-// Use only these methods to access or modify dialHistory.
-func (h dialHistory) min() pastDial {
-	return h[0]
-}
-func (h *dialHistory) add(id enode.ID, exp time.Time) {
-	heap.Push(h, pastDial{id, exp})
-
-}
-func (h *dialHistory) remove(id enode.ID) bool {
-	for i, v := range *h {
-		if v.id == id {
-			heap.Remove(h, i)
-			return true
-		}
-	}
-	return false
-}
-func (h dialHistory) contains(id enode.ID) bool {
-	for _, v := range h {
-		if v.id == id {
-			return true
-		}
-	}
-	return false
-}
-func (h *dialHistory) expire(now time.Time) {
-	for h.Len() > 0 && h.min().exp.Before(now) {
-		heap.Pop(h)
-	}
-}
-
-// heap.Interface boilerplate
-func (h dialHistory) Len() int           { return len(h) }
-func (h dialHistory) Less(i, j int) bool { return h[i].exp.Before(h[j].exp) }
-func (h dialHistory) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-func (h *dialHistory) Push(x interface{}) {
-	*h = append(*h, x.(pastDial))
-}
-func (h *dialHistory) Pop() interface{} {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[0 : n-1]
-	return x
-}

p2p/dial_test.go

@@ -20,10 +20,13 @@ import (
 	"encoding/binary"
 	"net"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/internal/testlog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/enr"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
@@ -67,10 +70,10 @@ func runDialTest(t *testing.T, test dialtest) {
 
 		new := test.init.newTasks(running, pm(round.peers), vtime)
 		if !sametasks(new, round.new) {
-			t.Errorf("round %d: new tasks mismatch:\ngot %v\nwant %v\nstate: %v\nrunning: %v\n",
+			t.Errorf("ERROR round %d: got %v\nwant %v\nstate: %v\nrunning: %v",
 				i, spew.Sdump(new), spew.Sdump(round.new), spew.Sdump(test.init), spew.Sdump(running))
 		}
-		t.Log("tasks:", spew.Sdump(new))
+		t.Logf("round %d new tasks: %s", i, strings.TrimSpace(spew.Sdump(new)))
 
 		// Time advances by 16 seconds on every round.
 		vtime = vtime.Add(16 * time.Second)
@@ -88,8 +91,9 @@ func (t fakeTable) ReadRandomNodes(buf []*enode.Node) int { return copy(buf, t)
 
 // This test checks that dynamic dials are launched from discovery results.
 func TestDialStateDynDial(t *testing.T) {
+	config := &Config{Logger: testlog.Logger(t, log.LvlTrace)}
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, nil, nil, fakeTable{}, 5, nil),
+		init: newDialState(enode.ID{}, fakeTable{}, 5, config),
 		rounds: []round{
 			// A discovery query is launched.
 			{
@@ -153,7 +157,7 @@ func TestDialStateDynDial(t *testing.T) {
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 				new: []task{
-					&waitExpireTask{Duration: 14 * time.Second},
+					&waitExpireTask{Duration: 19 * time.Second},
 				},
 			},
 			// In this round, the peer with id 2 drops off. The query
@@ -223,10 +227,13 @@ func TestDialStateDynDial(t *testing.T) {
 
 // Tests that bootnodes are dialed if no peers are connectd, but not otherwise.
 func TestDialStateDynDialBootnode(t *testing.T) {
-	bootnodes := []*enode.Node{
+	config := &Config{
-		newNode(uintID(1), nil),
+		BootstrapNodes: []*enode.Node{
-		newNode(uintID(2), nil),
+			newNode(uintID(1), nil),
-		newNode(uintID(3), nil),
+			newNode(uintID(2), nil),
+			newNode(uintID(3), nil),
+		},
+		Logger: testlog.Logger(t, log.LvlTrace),
 	}
 	table := fakeTable{
 		newNode(uintID(4), nil),
@@ -236,7 +243,7 @@ func TestDialStateDynDialBootnode(t *testing.T) {
 		newNode(uintID(8), nil),
 	}
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, nil, bootnodes, table, 5, nil),
+		init: newDialState(enode.ID{}, table, 5, config),
 		rounds: []round{
 			// 2 dynamic dials attempted, bootnodes pending fallback interval
 			{
@@ -259,25 +266,24 @@ func TestDialStateDynDialBootnode(t *testing.T) {
 			{
 				new: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 			},
 			// No dials succeed, 2nd bootnode is attempted
 			{
 				done: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 				new: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(2), nil)},
+					&dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)},
+					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 			},
 			// No dials succeed, 3rd bootnode is attempted
 			{
 				done: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(2), nil)},
+					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 				new: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)},
@@ -288,21 +294,19 @@ func TestDialStateDynDialBootnode(t *testing.T) {
 				done: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)},
 				},
-				new: []task{
+				new: []task{},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
-				},
 			},
 			// Random dial succeeds, no more bootnodes are attempted
 			{
+				new: []task{
+					&waitExpireTask{3 * time.Second},
+				},
 				peers: []*Peer{
 					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(4), nil)}},
 				},
 				done: []task{
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)},
 					&dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)},
-					&dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)},
 				},
 			},
 		},
@@ -324,7 +328,7 @@ func TestDialStateDynDialFromTable(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, nil, nil, table, 10, nil),
+		init: newDialState(enode.ID{}, table, 10, &Config{Logger: testlog.Logger(t, log.LvlTrace)}),
 		rounds: []round{
 			// 5 out of 8 of the nodes returned by ReadRandomNodes are dialed.
 			{
@@ -430,7 +434,7 @@ func TestDialStateNetRestrict(t *testing.T) {
 	restrict.Add("127.0.2.0/24")
 
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, nil, nil, table, 10, restrict),
+		init: newDialState(enode.ID{}, table, 10, &Config{NetRestrict: restrict}),
 		rounds: []round{
 			{
 				new: []task{
@@ -444,16 +448,18 @@ func TestDialStateNetRestrict(t *testing.T) {
 
 // This test checks that static dials are launched.
 func TestDialStateStaticDial(t *testing.T) {
-	wantStatic := []*enode.Node{
+	config := &Config{
-		newNode(uintID(1), nil),
+		StaticNodes: []*enode.Node{
-		newNode(uintID(2), nil),
+			newNode(uintID(1), nil),
-		newNode(uintID(3), nil),
+			newNode(uintID(2), nil),
-		newNode(uintID(4), nil),
+			newNode(uintID(3), nil),
-		newNode(uintID(5), nil),
+			newNode(uintID(4), nil),
+			newNode(uintID(5), nil),
+		},
+		Logger: testlog.Logger(t, log.LvlTrace),
 	}
-
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, fakeTable{}, 0, config),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -495,7 +501,7 @@ func TestDialStateStaticDial(t *testing.T) {
 					&dialTask{flags: staticDialedConn, dest: newNode(uintID(5), nil)},
 				},
 				new: []task{
-					&waitExpireTask{Duration: 14 * time.Second},
+					&waitExpireTask{Duration: 19 * time.Second},
 				},
 			},
 			// Wait a round for dial history to expire, no new tasks should spawn.
@@ -511,6 +517,9 @@ func TestDialStateStaticDial(t *testing.T) {
 			// If a static node is dropped, it should be immediately redialed,
 			// irrespective whether it was originally static or dynamic.
 			{
+				done: []task{
+					&waitExpireTask{Duration: 19 * time.Second},
+				},
 				peers: []*Peer{
 					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}},
 					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(3), nil)}},
@@ -518,67 +527,24 @@ func TestDialStateStaticDial(t *testing.T) {
 				},
 				new: []task{
 					&dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)},
-					&dialTask{flags: staticDialedConn, dest: newNode(uintID(4), nil)},
 				},
 			},
 		},
 	})
 }
 
-// This test checks that static peers will be redialed immediately if they were re-added to a static list.
-func TestDialStaticAfterReset(t *testing.T) {
-	wantStatic := []*enode.Node{
-		newNode(uintID(1), nil),
-		newNode(uintID(2), nil),
-	}
-
-	rounds := []round{
-		// Static dials are launched for the nodes that aren't yet connected.
-		{
-			peers: nil,
-			new: []task{
-				&dialTask{flags: staticDialedConn, dest: newNode(uintID(1), nil)},
-				&dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)},
-			},
-		},
-		// No new dial tasks, all peers are connected.
-		{
-			peers: []*Peer{
-				{rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}},
-				{rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}},
-			},
-			done: []task{
-				&dialTask{flags: staticDialedConn, dest: newNode(uintID(1), nil)},
-				&dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)},
-			},
-			new: []task{
-				&waitExpireTask{Duration: 30 * time.Second},
-			},
-		},
-	}
-	dTest := dialtest{
-		init:   newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
-		rounds: rounds,
-	}
-	runDialTest(t, dTest)
-	for _, n := range wantStatic {
-		dTest.init.removeStatic(n)
-		dTest.init.addStatic(n)
-	}
-	// without removing peers they will be considered recently dialed
-	runDialTest(t, dTest)
-}
-
 // This test checks that past dials are not retried for some time.
 func TestDialStateCache(t *testing.T) {
-	wantStatic := []*enode.Node{
+	config := &Config{
-		newNode(uintID(1), nil),
+		StaticNodes: []*enode.Node{
-		newNode(uintID(2), nil),
+			newNode(uintID(1), nil),
-		newNode(uintID(3), nil),
+			newNode(uintID(2), nil),
+			newNode(uintID(3), nil),
+		},
+		Logger: testlog.Logger(t, log.LvlTrace),
 	}
-
 	runDialTest(t, dialtest{
-		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, fakeTable{}, 0, config),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -606,28 +572,37 @@ func TestDialStateCache(t *testing.T) {
 			// entry to expire.
 			{
 				peers: []*Peer{
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}},
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}},
 				},
 				done: []task{
 					&dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)},
 				},
 				new: []task{
-					&waitExpireTask{Duration: 14 * time.Second},
+					&waitExpireTask{Duration: 19 * time.Second},
 				},
 			},
 			// Still waiting for node 3's entry to expire in the cache.
 			{
 				peers: []*Peer{
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}},
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}},
+				},
+			},
+			{
+				peers: []*Peer{
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}},
 				},
 			},
 			// The cache entry for node 3 has expired and is retried.
 			{
+				done: []task{
+					&waitExpireTask{Duration: 19 * time.Second},
+				},
 				peers: []*Peer{
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}},
-					{rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}},
+					{rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}},
 				},
 				new: []task{
 					&dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)},
@@ -638,9 +613,13 @@ func TestDialStateCache(t *testing.T) {
 }
 
 func TestDialResolve(t *testing.T) {
+	config := &Config{
+		Logger: testlog.Logger(t, log.LvlTrace),
+		Dialer: TCPDialer{&net.Dialer{Deadline: time.Now().Add(-5 * time.Minute)}},
+	}
 	resolved := newNode(uintID(1), net.IP{127, 0, 55, 234})
 	table := &resolveMock{answer: resolved}
-	state := newDialState(enode.ID{}, nil, nil, table, 0, nil)
+	state := newDialState(enode.ID{}, table, 0, config)
 
 	// Check that the task is generated with an incomplete ID.
 	dest := newNode(uintID(1), nil)
@@ -651,8 +630,7 @@ func TestDialResolve(t *testing.T) {
 	}
 
 	// Now run the task, it should resolve the ID once.
-	config := Config{Dialer: TCPDialer{&net.Dialer{Deadline: time.Now().Add(-5 * time.Minute)}}}
+	srv := &Server{ntab: table, log: config.Logger, Config: *config}
-	srv := &Server{ntab: table, Config: config}
 	tasks[0].Do(srv)
 	if !reflect.DeepEqual(table.resolveCalls, []*enode.Node{dest}) {
 		t.Fatalf("wrong resolve calls, got %v", table.resolveCalls)

p2p/metrics.go

@@ -25,18 +25,17 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/ethereum/go-ethereum/p2p/enode"
-
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 )
 
 const (
-	MetricsInboundConnects  = "p2p/InboundConnects"  // Name for the registered inbound connects meter
+	MetricsInboundTraffic   = "p2p/ingress" // Name for the registered inbound traffic meter
-	MetricsInboundTraffic   = "p2p/InboundTraffic"   // Name for the registered inbound traffic meter
+	MetricsOutboundTraffic  = "p2p/egress"  // Name for the registered outbound traffic meter
-	MetricsOutboundConnects = "p2p/OutboundConnects" // Name for the registered outbound connects meter
+	MetricsOutboundConnects = "p2p/dials"   // Name for the registered outbound connects meter
-	MetricsOutboundTraffic  = "p2p/OutboundTraffic"  // Name for the registered outbound traffic meter
+	MetricsInboundConnects  = "p2p/serves"  // Name for the registered inbound connects meter
 
 	MeteredPeerLimit = 1024 // This amount of peers are individually metered
 )
@@ -46,6 +45,7 @@ var (
 	ingressTrafficMeter = metrics.NewRegisteredMeter(MetricsInboundTraffic, nil)   // Meter metering the cumulative ingress traffic
 	egressConnectMeter  = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections
 	egressTrafficMeter  = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil)  // Meter metering the cumulative egress traffic
+	activePeerCounter   = metrics.NewRegisteredCounter("p2p/peers", nil)           // Gauge tracking the current peer count
 
 	PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/")  // Registry containing the peer ingress
 	PeerEgressRegistry  = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
@@ -124,6 +124,8 @@ func newMeteredConn(conn net.Conn, ingress bool, ip net.IP) net.Conn {
 	} else {
 		egressConnectMeter.Mark(1)
 	}
+	activePeerCounter.Inc(1)
+
 	return &meteredConn{
 		Conn:      conn,
 		ip:        ip,
@@ -198,6 +200,7 @@ func (c *meteredConn) Close() error {
 			IP:      c.ip,
 			Elapsed: time.Since(c.connected),
 		})
+		activePeerCounter.Dec(1)
 		return err
 	}
 	id := c.id
@@ -209,6 +212,7 @@ func (c *meteredConn) Close() error {
 			IP:   c.ip,
 			ID:   id,
 		})
+		activePeerCounter.Dec(1)
 		return err
 	}
 	ingress, egress := uint64(c.ingressMeter.Count()), uint64(c.egressMeter.Count())
@@ -229,5 +233,6 @@ func (c *meteredConn) Close() error {
 		Ingress: ingress,
 		Egress:  egress,
 	})
+	activePeerCounter.Dec(1)
 	return err
 }

p2p/netutil/addrutil.go

@@ -0,0 +1,33 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package netutil
+
+import "net"
+
+// AddrIP gets the IP address contained in addr. It returns nil if no address is present.
+func AddrIP(addr net.Addr) net.IP {
+	switch a := addr.(type) {
+	case *net.IPAddr:
+		return a.IP
+	case *net.TCPAddr:
+		return a.IP
+	case *net.UDPAddr:
+		return a.IP
+	default:
+		return nil
+	}
+}

p2p/peer.go

@@ -120,7 +120,7 @@ func NewPeer(id enode.ID, name string, caps []Cap) *Peer {
 	pipe, _ := net.Pipe()
 	node := enode.SignNull(new(enr.Record), id)
 	conn := &conn{fd: pipe, transport: nil, node: node, caps: caps, name: name}
-	peer := newPeer(conn, nil)
+	peer := newPeer(log.Root(), conn, nil)
 	close(peer.closed) // ensures Disconnect doesn't block
 	return peer
 }
@@ -176,7 +176,7 @@ func (p *Peer) Inbound() bool {
 	return p.rw.is(inboundConn)
 }
 
-func newPeer(conn *conn, protocols []Protocol) *Peer {
+func newPeer(log log.Logger, conn *conn, protocols []Protocol) *Peer {
 	protomap := matchProtocols(protocols, conn.caps, conn)
 	p := &Peer{
 		rw:       conn,

p2p/peer_test.go

@@ -24,6 +24,8 @@ import (
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/ethereum/go-ethereum/log"
 )
 
 var discard = Protocol{
@@ -52,7 +54,7 @@ func testPeer(protos []Protocol) (func(), *conn, *Peer, <-chan error) {
 		c2.caps = append(c2.caps, p.cap())
 	}
 
-	peer := newPeer(c1, protos)
+	peer := newPeer(log.Root(), c1, protos)
 	errc := make(chan error, 1)
 	go func() {
 		_, err := peer.run()

p2p/server.go

@@ -22,6 +22,7 @@ import (
 	"crypto/ecdsa"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"net"
 	"sort"
 	"sync"
@@ -49,6 +50,9 @@ const (
 	defaultMaxPendingPeers = 50
 	defaultDialRatio       = 3
 
+	// This time limits inbound connection attempts per source IP.
+	inboundThrottleTime = 30 * time.Second
+
 	// Maximum time allowed for reading a complete message.
 	// This is effectively the amount of time a connection can be idle.
 	frameReadTimeout = 30 * time.Second
@@ -158,6 +162,7 @@ type Server struct {
 	// the whole protocol stack.
 	newTransport func(net.Conn) transport
 	newPeerHook  func(*Peer)
+	listenFunc   func(network, addr string) (net.Listener, error)
 
 	lock    sync.Mutex // protects running
 	running bool
@@ -167,24 +172,26 @@ type Server struct {
 	ntab         discoverTable
 	listener     net.Listener
 	ourHandshake *protoHandshake
-	lastLookup   time.Time
 	DiscV5       *discv5.Network
+	loopWG       sync.WaitGroup // loop, listenLoop
+	peerFeed     event.Feed
+	log          log.Logger
 
-	// These are for Peers, PeerCount (and nothing else).
+	// Channels into the run loop.
-	peerOp     chan peerOpFunc
+	quit                    chan struct{}
-	peerOpDone chan struct{}
+	addstatic               chan *enode.Node
+	removestatic            chan *enode.Node
+	addtrusted              chan *enode.Node
+	removetrusted           chan *enode.Node
+	peerOp                  chan peerOpFunc
+	peerOpDone              chan struct{}
+	delpeer                 chan peerDrop
+	checkpointPostHandshake chan *conn
+	checkpointAddPeer       chan *conn
 
-	quit          chan struct{}
+	// State of run loop and listenLoop.
-	addstatic     chan *enode.Node
+	lastLookup     time.Time
-	removestatic  chan *enode.Node
+	inboundHistory expHeap
-	addtrusted    chan *enode.Node
-	removetrusted chan *enode.Node
-	posthandshake chan *conn
-	addpeer       chan *conn
-	delpeer       chan peerDrop
-	loopWG        sync.WaitGroup // loop, listenLoop
-	peerFeed      event.Feed
-	log           log.Logger
 }
 
 type peerOpFunc func(map[enode.ID]*Peer)
@@ -415,7 +422,7 @@ func (srv *Server) Start() (err error) {
 	srv.running = true
 	srv.log = srv.Config.Logger
 	if srv.log == nil {
-		srv.log = log.New()
+		srv.log = log.Root()
 	}
 	if srv.NoDial && srv.ListenAddr == "" {
 		srv.log.Warn("P2P server will be useless, neither dialing nor listening")
@@ -428,13 +435,16 @@ func (srv *Server) Start() (err error) {
 	if srv.newTransport == nil {
 		srv.newTransport = newRLPX
 	}
+	if srv.listenFunc == nil {
+		srv.listenFunc = net.Listen
+	}
 	if srv.Dialer == nil {
 		srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}}
 	}
 	srv.quit = make(chan struct{})
-	srv.addpeer = make(chan *conn)
 	srv.delpeer = make(chan peerDrop)
-	srv.posthandshake = make(chan *conn)
+	srv.checkpointPostHandshake = make(chan *conn)
+	srv.checkpointAddPeer = make(chan *conn)
 	srv.addstatic = make(chan *enode.Node)
 	srv.removestatic = make(chan *enode.Node)
 	srv.addtrusted = make(chan *enode.Node)
@@ -455,7 +465,7 @@ func (srv *Server) Start() (err error) {
 	}
 
 	dynPeers := srv.maxDialedConns()
-	dialer := newDialState(srv.localnode.ID(), srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict)
+	dialer := newDialState(srv.localnode.ID(), srv.ntab, dynPeers, &srv.Config)
 	srv.loopWG.Add(1)
 	go srv.run(dialer)
 	return nil
@@ -541,6 +551,7 @@ func (srv *Server) setupDiscovery() error {
 			NetRestrict: srv.NetRestrict,
 			Bootnodes:   srv.BootstrapNodes,
 			Unhandled:   unhandled,
+			Log:         srv.log,
 		}
 		ntab, err := discover.ListenUDP(conn, srv.localnode, cfg)
 		if err != nil {
@@ -569,27 +580,28 @@ func (srv *Server) setupDiscovery() error {
 }
 
 func (srv *Server) setupListening() error {
-	// Launch the TCP listener.
+	// Launch the listener.
-	listener, err := net.Listen("tcp", srv.ListenAddr)
+	listener, err := srv.listenFunc("tcp", srv.ListenAddr)
 	if err != nil {
 		return err
 	}
-	laddr := listener.Addr().(*net.TCPAddr)
-	srv.ListenAddr = laddr.String()
 	srv.listener = listener
-	srv.localnode.Set(enr.TCP(laddr.Port))
+	srv.ListenAddr = listener.Addr().String()
+
+	// Update the local node record and map the TCP listening port if NAT is configured.
+	if tcp, ok := listener.Addr().(*net.TCPAddr); ok {
+		srv.localnode.Set(enr.TCP(tcp.Port))
+		if !tcp.IP.IsLoopback() && srv.NAT != nil {
+			srv.loopWG.Add(1)
+			go func() {
+				nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p")
+				srv.loopWG.Done()
+			}()
+		}
+	}
 
 	srv.loopWG.Add(1)
 	go srv.listenLoop()
-
-	// Map the TCP listening port if NAT is configured.
-	if !laddr.IP.IsLoopback() && srv.NAT != nil {
-		srv.loopWG.Add(1)
-		go func() {
-			nat.Map(srv.NAT, srv.quit, "tcp", laddr.Port, laddr.Port, "ethereum p2p")
-			srv.loopWG.Done()
-		}()
-	}
 	return nil
 }
 
@@ -657,12 +669,14 @@ running:
 		case <-srv.quit:
 			// The server was stopped. Run the cleanup logic.
 			break running
+
 		case n := <-srv.addstatic:
 			// This channel is used by AddPeer to add to the
 			// ephemeral static peer list. Add it to the dialer,
 			// it will keep the node connected.
 			srv.log.Trace("Adding static node", "node", n)
 			dialstate.addStatic(n)
+
 		case n := <-srv.removestatic:
 			// This channel is used by RemovePeer to send a
 			// disconnect request to a peer and begin the
@@ -672,6 +686,7 @@ running:
 			if p, ok := peers[n.ID()]; ok {
 				p.Disconnect(DiscRequested)
 			}
+
 		case n := <-srv.addtrusted:
 			// This channel is used by AddTrustedPeer to add an enode
 			// to the trusted node set.
@@ -681,21 +696,23 @@ running:
 			if p, ok := peers[n.ID()]; ok {
 				p.rw.set(trustedConn, true)
 			}
+
 		case n := <-srv.removetrusted:
 			// This channel is used by RemoveTrustedPeer to remove an enode
 			// from the trusted node set.
 			srv.log.Trace("Removing trusted node", "node", n)
-			if _, ok := trusted[n.ID()]; ok {
+			delete(trusted, n.ID())
-				delete(trusted, n.ID())
+
-			}
 			// Unmark any already-connected peer as trusted
 			if p, ok := peers[n.ID()]; ok {
 				p.rw.set(trustedConn, false)
 			}
+
 		case op := <-srv.peerOp:
 			// This channel is used by Peers and PeerCount.
 			op(peers)
 			srv.peerOpDone <- struct{}{}
+
 		case t := <-taskdone:
 			// A task got done. Tell dialstate about it so it
 			// can update its state and remove it from the active
@@ -703,7 +720,8 @@ running:
 			srv.log.Trace("Dial task done", "task", t)
 			dialstate.taskDone(t, time.Now())
 			delTask(t)
-		case c := <-srv.posthandshake:
+
+		case c := <-srv.checkpointPostHandshake:
 			// A connection has passed the encryption handshake so
 			// the remote identity is known (but hasn't been verified yet).
 			if trusted[c.node.ID()] {
@@ -711,25 +729,22 @@ running:
 				c.flags |= trustedConn
 			}
 			// TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them.
-			select {
+			c.cont <- srv.postHandshakeChecks(peers, inboundCount, c)
-			case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c):
+
-			case <-srv.quit:
+		case c := <-srv.checkpointAddPeer:
-				break running
-			}
-		case c := <-srv.addpeer:
 			// At this point the connection is past the protocol handshake.
 			// Its capabilities are known and the remote identity is verified.
-			err := srv.protoHandshakeChecks(peers, inboundCount, c)
+			err := srv.addPeerChecks(peers, inboundCount, c)
 			if err == nil {
 				// The handshakes are done and it passed all checks.
-				p := newPeer(c, srv.Protocols)
+				p := newPeer(srv.log, c, srv.Protocols)
 				// If message events are enabled, pass the peerFeed
 				// to the peer
 				if srv.EnableMsgEvents {
 					p.events = &srv.peerFeed
 				}
 				name := truncateName(c.name)
-				srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1)
+				p.log.Debug("Adding p2p peer", "addr", p.RemoteAddr(), "peers", len(peers)+1, "name", name)
 				go srv.runPeer(p)
 				peers[c.node.ID()] = p
 				if p.Inbound() {
@@ -739,15 +754,12 @@ running:
 			// The dialer logic relies on the assumption that
 			// dial tasks complete after the peer has been added or
 			// discarded. Unblock the task last.
-			select {
+			c.cont <- err
-			case c.cont <- err:
+
-			case <-srv.quit:
-				break running
-			}
 		case pd := <-srv.delpeer:
 			// A peer disconnected.
 			d := common.PrettyDuration(mclock.Now() - pd.created)
-			pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err)
+			pd.log.Debug("Removing p2p peer", "addr", pd.RemoteAddr(), "peers", len(peers)-1, "duration", d, "req", pd.requested, "err", pd.err)
 			delete(peers, pd.ID())
 			if pd.Inbound() {
 				inboundCount--
@@ -778,17 +790,7 @@ running:
 	}
 }
 
-func (srv *Server) protoHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error {
+func (srv *Server) postHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error {
-	// Drop connections with no matching protocols.
-	if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 {
-		return DiscUselessPeer
-	}
-	// Repeat the encryption handshake checks because the
-	// peer set might have changed between the handshakes.
-	return srv.encHandshakeChecks(peers, inboundCount, c)
-}
-
-func (srv *Server) encHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error {
 	switch {
 	case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers:
 		return DiscTooManyPeers
@@ -803,9 +805,20 @@ func (srv *Server) encHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int
 	}
 }
 
+func (srv *Server) addPeerChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error {
+	// Drop connections with no matching protocols.
+	if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 {
+		return DiscUselessPeer
+	}
+	// Repeat the post-handshake checks because the
+	// peer set might have changed since those checks were performed.
+	return srv.postHandshakeChecks(peers, inboundCount, c)
+}
+
 func (srv *Server) maxInboundConns() int {
 	return srv.MaxPeers - srv.maxDialedConns()
 }
+
 func (srv *Server) maxDialedConns() int {
 	if srv.NoDiscovery || srv.NoDial {
 		return 0
@@ -833,7 +846,7 @@ func (srv *Server) listenLoop() {
 	}
 
 	for {
-		// Wait for a handshake slot before accepting.
+		// Wait for a free slot before accepting.
 		<-slots
 
 		var (
@@ -852,21 +865,16 @@ func (srv *Server) listenLoop() {
 			break
 		}
 
-		// Reject connections that do not match NetRestrict.
+		remoteIP := netutil.AddrIP(fd.RemoteAddr())
-		if srv.NetRestrict != nil {
+		if err := srv.checkInboundConn(fd, remoteIP); err != nil {
-			if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok && !srv.NetRestrict.Contains(tcp.IP) {
+			srv.log.Debug("Rejected inbound connnection", "addr", fd.RemoteAddr(), "err", err)
-				srv.log.Debug("Rejected conn (not whitelisted in NetRestrict)", "addr", fd.RemoteAddr())
+			fd.Close()
-				fd.Close()
+			slots <- struct{}{}
-				slots <- struct{}{}
+			continue
-				continue
-			}
 		}
-
+		if remoteIP != nil {
-		var ip net.IP
+			fd = newMeteredConn(fd, true, remoteIP)
-		if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok {
-			ip = tcp.IP
 		}
-		fd = newMeteredConn(fd, true, ip)
 		srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr())
 		go func() {
 			srv.SetupConn(fd, inboundConn, nil)
@@ -875,6 +883,22 @@ func (srv *Server) listenLoop() {
 	}
 }
 
+func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error {
+	if remoteIP != nil {
+		// Reject connections that do not match NetRestrict.
+		if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) {
+			return fmt.Errorf("not whitelisted in NetRestrict")
+		}
+		// Reject Internet peers that try too often.
+		srv.inboundHistory.expire(time.Now())
+		if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) {
+			return fmt.Errorf("too many attempts")
+		}
+		srv.inboundHistory.add(remoteIP.String(), time.Now().Add(inboundThrottleTime))
+	}
+	return nil
+}
+
 // SetupConn runs the handshakes and attempts to add the connection
 // as a peer. It returns when the connection has been added as a peer
 // or the handshakes have failed.
@@ -896,6 +920,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro
 	if !running {
 		return errServerStopped
 	}
+
 	// If dialing, figure out the remote public key.
 	var dialPubkey *ecdsa.PublicKey
 	if dialDest != nil {
@@ -904,7 +929,8 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro
 			return errors.New("dial destination doesn't have a secp256k1 public key")
 		}
 	}
-	// Run the encryption handshake.
+
+	// Run the RLPx handshake.
 	remotePubkey, err := c.doEncHandshake(srv.PrivateKey, dialPubkey)
 	if err != nil {
 		srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err)
@@ -923,12 +949,13 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro
 		conn.handshakeDone(c.node.ID())
 	}
 	clog := srv.log.New("id", c.node.ID(), "addr", c.fd.RemoteAddr(), "conn", c.flags)
-	err = srv.checkpoint(c, srv.posthandshake)
+	err = srv.checkpoint(c, srv.checkpointPostHandshake)
 	if err != nil {
-		clog.Trace("Rejected peer before protocol handshake", "err", err)
+		clog.Trace("Rejected peer", "err", err)
 		return err
 	}
-	// Run the protocol handshake
+
+	// Run the capability negotiation handshake.
 	phs, err := c.doProtoHandshake(srv.ourHandshake)
 	if err != nil {
 		clog.Trace("Failed proto handshake", "err", err)
@@ -939,14 +966,15 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro
 		return DiscUnexpectedIdentity
 	}
 	c.caps, c.name = phs.Caps, phs.Name
-	err = srv.checkpoint(c, srv.addpeer)
+	err = srv.checkpoint(c, srv.checkpointAddPeer)
 	if err != nil {
 		clog.Trace("Rejected peer", "err", err)
 		return err
 	}
-	// If the checks completed successfully, runPeer has now been
+
-	// launched by run.
+	// If the checks completed successfully, the connection has been added as a peer and
-	clog.Trace("connection set up", "inbound", dialDest == nil)
+	// runPeer has been launched.
+	clog.Trace("Connection set up", "inbound", dialDest == nil)
 	return nil
 }
 
@@ -975,12 +1003,7 @@ func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error {
 	case <-srv.quit:
 		return errServerStopped
 	}
-	select {
+	return <-c.cont
-	case err := <-c.cont:
-		return err
-	case <-srv.quit:
-		return errServerStopped
-	}
 }
 
 // runPeer runs in its own goroutine for each peer.

p2p/server_test.go

@@ -19,6 +19,7 @@ package p2p
 import (
 	"crypto/ecdsa"
 	"errors"
+	"io"
 	"math/rand"
 	"net"
 	"reflect"
@@ -26,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/testlog"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/enr"
@@ -74,6 +76,7 @@ func startTestServer(t *testing.T, remoteKey *ecdsa.PublicKey, pf func(*Peer)) *
 		MaxPeers:   10,
 		ListenAddr: "127.0.0.1:0",
 		PrivateKey: newkey(),
+		Logger:     testlog.Logger(t, log.LvlTrace),
 	}
 	server := &Server{
 		Config:       config,
@@ -359,6 +362,7 @@ func TestServerAtCap(t *testing.T) {
 			PrivateKey:   newkey(),
 			MaxPeers:     10,
 			NoDial:       true,
+			NoDiscovery:  true,
 			TrustedNodes: []*enode.Node{newNode(trustedID, nil)},
 		},
 	}
@@ -377,19 +381,19 @@ func TestServerAtCap(t *testing.T) {
 	// Inject a few connections to fill up the peer set.
 	for i := 0; i < 10; i++ {
 		c := newconn(randomID())
-		if err := srv.checkpoint(c, srv.addpeer); err != nil {
+		if err := srv.checkpoint(c, srv.checkpointAddPeer); err != nil {
 			t.Fatalf("could not add conn %d: %v", i, err)
 		}
 	}
 	// Try inserting a non-trusted connection.
 	anotherID := randomID()
 	c := newconn(anotherID)
-	if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers {
+	if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != DiscTooManyPeers {
 		t.Error("wrong error for insert:", err)
 	}
 	// Try inserting a trusted connection.
 	c = newconn(trustedID)
-	if err := srv.checkpoint(c, srv.posthandshake); err != nil {
+	if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != nil {
 		t.Error("unexpected error for trusted conn @posthandshake:", err)
 	}
 	if !c.is(trustedConn) {
@@ -399,14 +403,14 @@ func TestServerAtCap(t *testing.T) {
 	// Remove from trusted set and try again
 	srv.RemoveTrustedPeer(newNode(trustedID, nil))
 	c = newconn(trustedID)
-	if err := srv.checkpoint(c, srv.posthandshake); err != DiscTooManyPeers {
+	if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != DiscTooManyPeers {
 		t.Error("wrong error for insert:", err)
 	}
 
 	// Add anotherID to trusted set and try again
 	srv.AddTrustedPeer(newNode(anotherID, nil))
 	c = newconn(anotherID)
-	if err := srv.checkpoint(c, srv.posthandshake); err != nil {
+	if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != nil {
 		t.Error("unexpected error for trusted conn @posthandshake:", err)
 	}
 	if !c.is(trustedConn) {
@@ -430,10 +434,11 @@ func TestServerPeerLimits(t *testing.T) {
 
 	srv := &Server{
 		Config: Config{
-			PrivateKey: srvkey,
+			PrivateKey:  srvkey,
-			MaxPeers:   0,
+			MaxPeers:    0,
-			NoDial:     true,
+			NoDial:      true,
-			Protocols:  []Protocol{discard},
+			NoDiscovery: true,
+			Protocols:   []Protocol{discard},
 		},
 		newTransport: func(fd net.Conn) transport { return tp },
 		log:          log.New(),
@@ -541,29 +546,35 @@ func TestServerSetupConn(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		srv := &Server{
+		t.Run(test.wantCalls, func(t *testing.T) {
-			Config: Config{
+			cfg := Config{
-				PrivateKey: srvkey,
+				PrivateKey:  srvkey,
-				MaxPeers:   10,
+				MaxPeers:    10,
-				NoDial:     true,
+				NoDial:      true,
-				Protocols:  []Protocol{discard},
+				NoDiscovery: true,
-			},
+				Protocols:   []Protocol{discard},
-			newTransport: func(fd net.Conn) transport { return test.tt },
+				Logger:      testlog.Logger(t, log.LvlTrace),
-			log:          log.New(),
-		}
-		if !test.dontstart {
-			if err := srv.Start(); err != nil {
-				t.Fatalf("couldn't start server: %v", err)
 			}
-		}
+			srv := &Server{
-		p1, _ := net.Pipe()
+				Config:       cfg,
-		srv.SetupConn(p1, test.flags, test.dialDest)
+				newTransport: func(fd net.Conn) transport { return test.tt },
-		if !reflect.DeepEqual(test.tt.closeErr, test.wantCloseErr) {
+				log:          cfg.Logger,
-			t.Errorf("test %d: close error mismatch: got %q, want %q", i, test.tt.closeErr, test.wantCloseErr)
+			}
-		}
+			if !test.dontstart {
-		if test.tt.calls != test.wantCalls {
+				if err := srv.Start(); err != nil {
-			t.Errorf("test %d: calls mismatch: got %q, want %q", i, test.tt.calls, test.wantCalls)
+					t.Fatalf("couldn't start server: %v", err)
-		}
+				}
+				defer srv.Stop()
+			}
+			p1, _ := net.Pipe()
+			srv.SetupConn(p1, test.flags, test.dialDest)
+			if !reflect.DeepEqual(test.tt.closeErr, test.wantCloseErr) {
+				t.Errorf("test %d: close error mismatch: got %q, want %q", i, test.tt.closeErr, test.wantCloseErr)
+			}
+			if test.tt.calls != test.wantCalls {
+				t.Errorf("test %d: calls mismatch: got %q, want %q", i, test.tt.calls, test.wantCalls)
+			}
+		})
 	}
 }
 
@@ -616,3 +627,100 @@ func randomID() (id enode.ID) {
 	}
 	return id
 }
+
+// This test checks that inbound connections are throttled by IP.
+func TestServerInboundThrottle(t *testing.T) {
+	const timeout = 5 * time.Second
+	newTransportCalled := make(chan struct{})
+	srv := &Server{
+		Config: Config{
+			PrivateKey:  newkey(),
+			ListenAddr:  "127.0.0.1:0",
+			MaxPeers:    10,
+			NoDial:      true,
+			NoDiscovery: true,
+			Protocols:   []Protocol{discard},
+			Logger:      testlog.Logger(t, log.LvlTrace),
+		},
+		newTransport: func(fd net.Conn) transport {
+			newTransportCalled <- struct{}{}
+			return newRLPX(fd)
+		},
+		listenFunc: func(network, laddr string) (net.Listener, error) {
+			fakeAddr := &net.TCPAddr{IP: net.IP{95, 33, 21, 2}, Port: 4444}
+			return listenFakeAddr(network, laddr, fakeAddr)
+		},
+	}
+	if err := srv.Start(); err != nil {
+		t.Fatal("can't start: ", err)
+	}
+	defer srv.Stop()
+
+	// Dial the test server.
+	conn, err := net.DialTimeout("tcp", srv.ListenAddr, timeout)
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	select {
+	case <-newTransportCalled:
+		// OK
+	case <-time.After(timeout):
+		t.Error("newTransport not called")
+	}
+	conn.Close()
+
+	// Dial again. This time the server should close the connection immediately.
+	connClosed := make(chan struct{})
+	conn, err = net.DialTimeout("tcp", srv.ListenAddr, timeout)
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	defer conn.Close()
+	go func() {
+		conn.SetDeadline(time.Now().Add(timeout))
+		buf := make([]byte, 10)
+		if n, err := conn.Read(buf); err != io.EOF || n != 0 {
+			t.Errorf("expected io.EOF and n == 0, got error %q and n == %d", err, n)
+		}
+		connClosed <- struct{}{}
+	}()
+	select {
+	case <-connClosed:
+		// OK
+	case <-newTransportCalled:
+		t.Error("newTransport called for second attempt")
+	case <-time.After(timeout):
+		t.Error("connection not closed within timeout")
+	}
+}
+
+func listenFakeAddr(network, laddr string, remoteAddr net.Addr) (net.Listener, error) {
+	l, err := net.Listen(network, laddr)
+	if err == nil {
+		l = &fakeAddrListener{l, remoteAddr}
+	}
+	return l, err
+}
+
+// fakeAddrListener is a listener that creates connections with a mocked remote address.
+type fakeAddrListener struct {
+	net.Listener
+	remoteAddr net.Addr
+}
+
+type fakeAddrConn struct {
+	net.Conn
+	remoteAddr net.Addr
+}
+
+func (l *fakeAddrListener) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	return &fakeAddrConn{c, l.remoteAddr}, nil
+}
+
+func (c *fakeAddrConn) RemoteAddr() net.Addr {
+	return c.remoteAddr
+}

p2p/util.go

@@ -0,0 +1,82 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package p2p
+
+import (
+	"container/heap"
+	"time"
+)
+
+// expHeap tracks strings and their expiry time.
+type expHeap []expItem
+
+// expItem is an entry in addrHistory.
+type expItem struct {
+	item string
+	exp  time.Time
+}
+
+// nextExpiry returns the next expiry time.
+func (h *expHeap) nextExpiry() time.Time {
+	return (*h)[0].exp
+}
+
+// add adds an item and sets its expiry time.
+func (h *expHeap) add(item string, exp time.Time) {
+	heap.Push(h, expItem{item, exp})
+}
+
+// remove removes an item.
+func (h *expHeap) remove(item string) bool {
+	for i, v := range *h {
+		if v.item == item {
+			heap.Remove(h, i)
+			return true
+		}
+	}
+	return false
+}
+
+// contains checks whether an item is present.
+func (h expHeap) contains(item string) bool {
+	for _, v := range h {
+		if v.item == item {
+			return true
+		}
+	}
+	return false
+}
+
+// expire removes items with expiry time before 'now'.
+func (h *expHeap) expire(now time.Time) {
+	for h.Len() > 0 && h.nextExpiry().Before(now) {
+		heap.Pop(h)
+	}
+}
+
+// heap.Interface boilerplate
+func (h expHeap) Len() int            { return len(h) }
+func (h expHeap) Less(i, j int) bool  { return h[i].exp.Before(h[j].exp) }
+func (h expHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
+func (h *expHeap) Push(x interface{}) { *h = append(*h, x.(expItem)) }
+func (h *expHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}

p2p/util_test.go

@@ -0,0 +1,54 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package p2p
+
+import (
+	"testing"
+	"time"
+)
+
+func TestExpHeap(t *testing.T) {
+	var h expHeap
+
+	var (
+		basetime = time.Unix(4000, 0)
+		exptimeA = basetime.Add(2 * time.Second)
+		exptimeB = basetime.Add(3 * time.Second)
+		exptimeC = basetime.Add(4 * time.Second)
+	)
+	h.add("a", exptimeA)
+	h.add("b", exptimeB)
+	h.add("c", exptimeC)
+
+	if !h.nextExpiry().Equal(exptimeA) {
+		t.Fatal("wrong nextExpiry")
+	}
+	if !h.contains("a") || !h.contains("b") || !h.contains("c") {
+		t.Fatal("heap doesn't contain all live items")
+	}
+
+	h.expire(exptimeA.Add(1))
+	if !h.nextExpiry().Equal(exptimeB) {
+		t.Fatal("wrong nextExpiry")
+	}
+	if h.contains("a") {
+		t.Fatal("heap contains a even though it has already expired")
+	}
+	if !h.contains("b") || !h.contains("c") {
+		t.Fatal("heap doesn't contain all live items")
+	}
+}

vendor/github.com/karalabe/usb/appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.12.5.windows-%GOARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.12.6.windows-%GOARCH%.zip
-  - 7z x go1.12.5.windows-%GOARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.12.6.windows-%GOARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

vendor/vendor.json

@@ -243,10 +243,10 @@
 			"revisionTime": "2017-04-30T22:20:11Z"
 		},
 		{
-			"checksumSHA1": "wqKuSmlxa4hVZwfs4ZjtKtM5Yp0=",
+			"checksumSHA1": "TU/WaqL7fYPDovmGVRSo8btD4ZM=",
 			"path": "github.com/karalabe/usb",
-			"revision": "3eff4807c21acef1c5f1b712a225ab09a8961504",
+			"revision": "4d6ba34a841453237dba721eeec1af0ef9a7a890",
-			"revisionTime": "2019-06-07T09:24:32Z",
+			"revisionTime": "2019-06-13T07:02:55Z",
 			"tree": true
 		},
 		{