forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #7 from ethereum/master

Browse source 
update
This commit is contained in:
jkcomment 2020-04-20 21:26:20 +09:00 committed by GitHub
commit f1134257f0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
463 changed files with 23562 additions and 6951 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE.md vendored
View file

@ -2,7 +2,7 @@ Hi there,
Please note that this is an issue tracker reserved for bug reports and feature requests.
For general questions please use the gitter channel or the Ethereum stack exchange at https://ethereum.stackexchange.com.
For general questions please use [discord](https://discord.gg/nthXNEv) or the Ethereum stack exchange at https://ethereum.stackexchange.com.
#### System information

55
.travis.yml
View file

@ -2,12 +2,21 @@ language: go
go_import_path: github.com/ethereum/go-ethereum
sudo: false
jobs:
allow_failures:
- stage: build
os: osx
go: 1.14.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
include:
# This builder only tests code linters on latest version of Go
# This builder only tests code linters on latest version of Go
- stage: lint
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- lint
git:
@ -35,12 +44,24 @@ jobs:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
os: linux
dist: xenial
go: 1.13.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
# These are the latest Go versions.
- stage: build
os: linux
arch: amd64
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
@ -50,7 +71,9 @@ jobs:
os: linux
arch: arm64
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
@ -58,7 +81,9 @@ jobs:
- stage: build
os: osx
osx_image: xcode11.3
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- echo "Increase the maximum number of open file descriptors on macOS"
- NOFILE=20480
@ -77,9 +102,10 @@ jobs:
if: type = push
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- ubuntu-ppa
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
@ -93,7 +119,7 @@ jobs:
- python-paramiko
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -goversion 1.13.6 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
- go run build/ci.go debsrc -goversion 1.14.2 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- stage: build
@ -101,9 +127,10 @@ jobs:
os: linux
dist: xenial
sudo: required
go: 1.13.x
go: 1.14.x
env:
- azure-linux
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
@ -137,9 +164,10 @@ jobs:
dist: xenial
services:
- docker
go: 1.13.x
go: 1.14.x
env:
- azure-linux-mips
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
@ -180,10 +208,11 @@ jobs:
env:
- azure-android
- maven-android
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- curl https://dl.google.com/go/go1.13.linux-amd64.tar.gz | tar -xz
- curl https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
@ -201,11 +230,12 @@ jobs:
- stage: build
if: type = push
os: osx
go: 1.13.x
go: 1.14.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
@ -232,9 +262,10 @@ jobs:
if: type = cron
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- azure-purge
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:

2
Dockerfile
View file

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.13-alpine as builder
FROM golang:1.14-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers git

2
Dockerfile.alltools
View file

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.13-alpine as builder
FROM golang:1.14-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers git

47
README.md
View file

@ -16,7 +16,7 @@ archives are published at https://geth.ethereum.org/downloads/.
For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki.
Building `geth` requires both a Go (version 1.10 or later) and a C compiler. You can install
Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install
them using your favourite package manager. Once the dependencies are installed, run
```shell
@ -39,7 +39,7 @@ directory.
| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). |
| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. |
@ -72,7 +72,7 @@ This command will:
This tool is optional and if you leave it out you can always attach to an already running
`geth` instance with `geth attach`.
### A Full node on the Ethereum test network
### A Full node on the Görli test network
Transitioning towards developers, if you'd like to play around with creating Ethereum
contracts, you almost certainly would like to do that without any real money involved until
@ -81,23 +81,24 @@ network, you want to join the **test** network with your node, which is fully eq
the main network, but with play-Ether only.
```shell
$ geth --testnet console
$ geth --goerli console
```
The `console` subcommand has the exact same meaning as above and they are equally
useful on the testnet too. Please see above for their explanations if you've skipped here.
useful on the testnet too. Please, see above for their explanations if you've skipped here.
Specifying the `--testnet` flag, however, will reconfigure your `geth` instance a bit:
Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit:
* Instead of connecting the main Ethereum network, the client will connect to the Görli
test network, which uses different P2P bootnodes, different network IDs and genesis
states.
* Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth`
will nest itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on
will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on
Linux). Note, on OSX and Linux this also means that attaching to a running testnet node
requires the use of a custom endpoint since `geth attach` will try to attach to a
production node endpoint by default. E.g.
`geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by
production node endpoint by default, e.g.,
`geth attach <datadir>/goerli/geth.ipc`. Windows users are not affected by
this.
* Instead of connecting the main Ethereum network, the client will connect to the test
network, which uses different P2P bootnodes, different network IDs and genesis states.
*Note: Although there are some internal protective measures to prevent transactions from
crossing over between the main network and test network, you should make sure to always
@ -107,17 +108,26 @@ accounts available between them.*
### Full node on the Rinkeby test network
The above test network is a cross-client one based on the ethash proof-of-work consensus
algorithm. As such, it has certain extra overhead and is more susceptible to reorganization
attacks due to the network's low difficulty/security. Go Ethereum also supports connecting
to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io)
(operated by members of the community). This network is lighter, more secure, but is only
supported by go-ethereum.
Go Ethereum also supports connecting to the older proof-of-authority based test network
called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community.
```shell
$ geth --rinkeby console
```
### Full node on the Ropsten test network
In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The
Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such,
it has certain extra overhead and is more susceptible to reorganization attacks due to the
network's low difficulty/security.
```shell
$ geth --ropsten console
```
*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.*
### Configuration
As an alternative to passing the numerous flags to the `geth` binary, you can also pass a
@ -217,7 +227,8 @@ aware of and agree upon. This consists of a small JSON file (e.g. call it `genes
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0
"petersburgBlock": 0,
"istanbulBlock": 0
},
"alloc": {},
"coinbase": "0x0000000000000000000000000000000000000000",

126
accounts/abi/abi.go
View file

@ -19,6 +19,7 @@ package abi
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
@ -32,6 +33,12 @@ type ABI struct {
Constructor Method
Methods map[string]Method
Events map[string]Event
// Additional "special" functions introduced in solidity v0.6.0.
// It's separated from the original default fallback. Each contract
// can only define one fallback and receive function.
Fallback Method // Note it's also used to represent legacy fallback before v0.6.0
Receive Method
}
// JSON returns a parsed ABI interface and error if it failed.
@ -42,7 +49,6 @@ func JSON(reader io.Reader) (ABI, error) {
if err := dec.Decode(&abi); err != nil {
return ABI{}, err
}
return abi, nil
}
@ -70,7 +76,7 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
return nil, err
}
// Pack up the method ID too if not a constructor and return
return append(method.ID(), arguments...), nil
return append(method.ID, arguments...), nil
}
// Unpack output in v according to the abi specification
@ -108,13 +114,22 @@ func (abi ABI) UnpackIntoMap(v map[string]interface{}, name string, data []byte)
// UnmarshalJSON implements json.Unmarshaler interface
func (abi *ABI) UnmarshalJSON(data []byte) error {
var fields []struct {
Type string
Name string
Constant bool
Type string
Name string
Inputs []Argument
Outputs []Argument
// Status indicator which can be: "pure", "view",
// "nonpayable" or "payable".
StateMutability string
Anonymous bool
Inputs []Argument
Outputs []Argument
// Deprecated Status indicators, but removed in v0.6.0.
Constant bool // True if function is either pure or view
Payable bool // True if function is payable
// Event relevant indicator represents the event is
// declared as anonymous.
Anonymous bool
}
if err := json.Unmarshal(data, &fields); err != nil {
return err
@ -124,44 +139,67 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
for _, field := range fields {
switch field.Type {
case "constructor":
abi.Constructor = Method{
Inputs: field.Inputs,
abi.Constructor = NewMethod("", "", Constructor, field.StateMutability, field.Constant, field.Payable, field.Inputs, nil)
case "function":
name := abi.overloadedMethodName(field.Name)
abi.Methods[name] = NewMethod(name, field.Name, Function, field.StateMutability, field.Constant, field.Payable, field.Inputs, field.Outputs)
case "fallback":
// New introduced function type in v0.6.0, check more detail
// here https://solidity.readthedocs.io/en/v0.6.0/contracts.html#fallback-function
if abi.HasFallback() {
return errors.New("only single fallback is allowed")
}
// empty defaults to function according to the abi spec
case "function", "":
name := field.Name
_, ok := abi.Methods[name]
for idx := 0; ok; idx++ {
name = fmt.Sprintf("%s%d", field.Name, idx)
_, ok = abi.Methods[name]
abi.Fallback = NewMethod("", "", Fallback, field.StateMutability, field.Constant, field.Payable, nil, nil)
case "receive":
// New introduced function type in v0.6.0, check more detail
// here https://solidity.readthedocs.io/en/v0.6.0/contracts.html#fallback-function
if abi.HasReceive() {
return errors.New("only single receive is allowed")
}
isConst := field.Constant || field.StateMutability == "pure" || field.StateMutability == "view"
abi.Methods[name] = Method{
Name: name,
RawName: field.Name,
Const: isConst,
Inputs: field.Inputs,
Outputs: field.Outputs,
if field.StateMutability != "payable" {
return errors.New("the statemutability of receive can only be payable")
}
abi.Receive = NewMethod("", "", Receive, field.StateMutability, field.Constant, field.Payable, nil, nil)
case "event":
name := field.Name
_, ok := abi.Events[name]
for idx := 0; ok; idx++ {
name = fmt.Sprintf("%s%d", field.Name, idx)
_, ok = abi.Events[name]
}
abi.Events[name] = Event{
Name: name,
RawName: field.Name,
Anonymous: field.Anonymous,
Inputs: field.Inputs,
}
name := abi.overloadedEventName(field.Name)
abi.Events[name] = NewEvent(name, field.Name, field.Anonymous, field.Inputs)
default:
return fmt.Errorf("abi: could not recognize type %v of field %v", field.Type, field.Name)
}
}
return nil
}
// overloadedMethodName returns the next available name for a given function.
// Needed since solidity allows for function overload.
//
// e.g. if the abi contains Methods send, send1
// overloadedMethodName would return send2 for input send.
func (abi *ABI) overloadedMethodName(rawName string) string {
name := rawName
_, ok := abi.Methods[name]
for idx := 0; ok; idx++ {
name = fmt.Sprintf("%s%d", rawName, idx)
_, ok = abi.Methods[name]
}
return name
}
// overloadedEventName returns the next available name for a given event.
// Needed since solidity allows for event overload.
//
// e.g. if the abi contains events received, received1
// overloadedEventName would return received2 for input received.
func (abi *ABI) overloadedEventName(rawName string) string {
name := rawName
_, ok := abi.Events[name]
for idx := 0; ok; idx++ {
name = fmt.Sprintf("%s%d", rawName, idx)
_, ok = abi.Events[name]
}
return name
}
// MethodById looks up a method by the 4-byte id
// returns nil if none found
func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
@ -169,7 +207,7 @@ func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
return nil, fmt.Errorf("data too short (%d bytes) for abi method lookup", len(sigdata))
}
for _, method := range abi.Methods {
if bytes.Equal(method.ID(), sigdata[:4]) {
if bytes.Equal(method.ID, sigdata[:4]) {
return &method, nil
}
}
@ -180,9 +218,19 @@ func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
// ABI and returns nil if none found.
func (abi *ABI) EventByID(topic common.Hash) (*Event, error) {
for _, event := range abi.Events {
if bytes.Equal(event.ID().Bytes(), topic.Bytes()) {
if bytes.Equal(event.ID.Bytes(), topic.Bytes()) {
return &event, nil
}
}
return nil, fmt.Errorf("no event with id: %#x", topic.Hex())
}
// HasFallback returns an indicator whether a fallback function is included.
func (abi *ABI) HasFallback() bool {
return abi.Fallback.Type == Fallback
}
// HasReceive returns an indicator whether a receive function is included.
func (abi *ABI) HasReceive() bool {
return abi.Receive.Type == Receive
}

117
accounts/abi/abi_test.go
View file

@ -31,47 +31,41 @@ import (
const jsondata = `
[
{ "type" : "function", "name" : "balance", "constant" : true },
{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }
{ "type" : "function", "name" : "balance", "stateMutability" : "view" },
{ "type" : "function", "name" : "send", "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }
]`
const jsondata2 = `
[
{ "type" : "function", "name" : "balance", "constant" : true },
{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
{ "type" : "function", "name" : "test", "constant" : false, "inputs" : [ { "name" : "number", "type" : "uint32" } ] },
{ "type" : "function", "name" : "string", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "string" } ] },
{ "type" : "function", "name" : "bool", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "bool" } ] },
{ "type" : "function", "name" : "address", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address" } ] },
{ "type" : "function", "name" : "uint64[2]", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] },
{ "type" : "function", "name" : "uint64[]", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] },
{ "type" : "function", "name" : "foo", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] },
{ "type" : "function", "name" : "bar", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] },
{ "type" : "function", "name" : "slice", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
{ "type" : "function", "name" : "slice256", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] },
{ "type" : "function", "name" : "sliceAddress", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] },
{ "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray2", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] },
{ "type" : "function", "name" : "nestedSlice", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] }
{ "type" : "function", "name" : "balance", "stateMutability" : "view" },
{ "type" : "function", "name" : "send", "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
{ "type" : "function", "name" : "test", "inputs" : [ { "name" : "number", "type" : "uint32" } ] },
{ "type" : "function", "name" : "string", "inputs" : [ { "name" : "inputs", "type" : "string" } ] },
{ "type" : "function", "name" : "bool", "inputs" : [ { "name" : "inputs", "type" : "bool" } ] },
{ "type" : "function", "name" : "address", "inputs" : [ { "name" : "inputs", "type" : "address" } ] },
{ "type" : "function", "name" : "uint64[2]", "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] },
{ "type" : "function", "name" : "uint64[]", "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] },
{ "type" : "function", "name" : "foo", "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] },
{ "type" : "function", "name" : "bar", "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] },
{ "type" : "function", "name" : "slice", "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
{ "type" : "function", "name" : "slice256", "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] },
{ "type" : "function", "name" : "sliceAddress", "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] },
{ "type" : "function", "name" : "sliceMultiAddress", "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray", "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray2", "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] },
{ "type" : "function", "name" : "nestedSlice", "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] }
]`
func TestReader(t *testing.T) {
Uint256, _ := NewType("uint256", "", nil)
exp := ABI{
abi := ABI{
Methods: map[string]Method{
"balance": {
"balance", "balance", true, nil, nil,
},
"send": {
"send", "send", false, []Argument{
{"amount", Uint256, false},
}, nil,
},
"balance": NewMethod("balance", "balance", Function, "view", false, false, nil, nil),
"send": NewMethod("send", "send", Function, "", false, false, []Argument{{"amount", Uint256, false}}, nil),
},
}
abi, err := JSON(strings.NewReader(jsondata))
exp, err := JSON(strings.NewReader(jsondata))
if err != nil {
t.Error(err)
}
@ -173,22 +167,22 @@ func TestTestSlice(t *testing.T) {
func TestMethodSignature(t *testing.T) {
String, _ := NewType("string", "", nil)
m := Method{"foo", "foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
m := NewMethod("foo", "foo", Function, "", false, false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil)
exp := "foo(string,string)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
if m.Sig != exp {
t.Error("signature mismatch", exp, "!=", m.Sig)
}
idexp := crypto.Keccak256([]byte(exp))[:4]
if !bytes.Equal(m.ID(), idexp) {
t.Errorf("expected ids to match %x != %x", m.ID(), idexp)
if !bytes.Equal(m.ID, idexp) {
t.Errorf("expected ids to match %x != %x", m.ID, idexp)
}
uintt, _ := NewType("uint256", "", nil)
m = Method{"foo", "foo", false, []Argument{{"bar", uintt, false}}, nil}
m = NewMethod("foo", "foo", Function, "", false, false, []Argument{{"bar", uintt, false}}, nil)
exp = "foo(uint256)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
if m.Sig != exp {
t.Error("signature mismatch", exp, "!=", m.Sig)
}
// Method with tuple arguments
@ -204,10 +198,10 @@ func TestMethodSignature(t *testing.T) {
{Name: "y", Type: "int256"},
}},
})
m = Method{"foo", "foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
m = NewMethod("foo", "foo", Function, "", false, false, []Argument{{"s", s, false}, {"bar", String, false}}, nil)
exp = "foo((int256,int256[],(int256,int256)[],(int256,int256)[2]),string)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
if m.Sig != exp {
t.Error("signature mismatch", exp, "!=", m.Sig)
}
}
@ -219,12 +213,12 @@ func TestOverloadedMethodSignature(t *testing.T) {
}
check := func(name string, expect string, method bool) {
if method {
if abi.Methods[name].Sig() != expect {
t.Fatalf("The signature of overloaded method mismatch, want %s, have %s", expect, abi.Methods[name].Sig())
if abi.Methods[name].Sig != expect {
t.Fatalf("The signature of overloaded method mismatch, want %s, have %s", expect, abi.Methods[name].Sig)
}
} else {
if abi.Events[name].Sig() != expect {
t.Fatalf("The signature of overloaded event mismatch, want %s, have %s", expect, abi.Events[name].Sig())
if abi.Events[name].Sig != expect {
t.Fatalf("The signature of overloaded event mismatch, want %s, have %s", expect, abi.Events[name].Sig)
}
}
}
@ -582,7 +576,7 @@ func TestInputFixedArrayAndVariableInputLength(t *testing.T) {
}
func TestDefaultFunctionParsing(t *testing.T) {
const definition = `[{ "name" : "balance" }]`
const definition = `[{ "name" : "balance", "type" : "function" }]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
@ -921,13 +915,13 @@ func TestABI_MethodById(t *testing.T) {
}
for name, m := range abi.Methods {
a := fmt.Sprintf("%v", m)
m2, err := abi.MethodById(m.ID())
m2, err := abi.MethodById(m.ID)
if err != nil {
t.Fatalf("Failed to look up ABI method: %v", err)
}
b := fmt.Sprintf("%v", m2)
if a != b {
t.Errorf("Method %v (id %x) not 'findable' by id in ABI", name, m.ID())
t.Errorf("Method %v (id %x) not 'findable' by id in ABI", name, m.ID)
}
}
// Also test empty
@ -995,8 +989,8 @@ func TestABI_EventById(t *testing.T) {
t.Errorf("We should find a event for topic %s, test #%d", topicID.Hex(), testnum)
}
if event.ID() != topicID {
t.Errorf("Event id %s does not match topic %s, test #%d", event.ID().Hex(), topicID.Hex(), testnum)
if event.ID != topicID {
t.Errorf("Event id %s does not match topic %s, test #%d", event.ID.Hex(), topicID.Hex(), testnum)
}
unknowntopicID := crypto.Keccak256Hash([]byte("unknownEvent"))
@ -1051,3 +1045,28 @@ func TestDoubleDuplicateMethodNames(t *testing.T) {
t.Fatalf("Should not have found extra method")
}
}
// TestUnnamedEventParam checks that an event with unnamed parameters is
// correctly handled
// The test runs the abi of the following contract.
// contract TestEvent {
// event send(uint256, uint256);
// }
func TestUnnamedEventParam(t *testing.T) {
abiJSON := `[{ "anonymous": false, "inputs": [{ "indexed": false,"internalType": "uint256", "name": "","type": "uint256"},{"indexed": false,"internalType": "uint256","name": "","type": "uint256"}],"name": "send","type": "event"}]`
contractAbi, err := JSON(strings.NewReader(abiJSON))
if err != nil {
t.Fatal(err)
}
event, ok := contractAbi.Events["send"]
if !ok {
t.Fatalf("Could not find event")
}
if event.Inputs[0].Name != "arg0" {
t.Fatalf("Could not find input")
}
if event.Inputs[1].Name != "arg1" {
t.Fatalf("Could not find input")
}
}

57
accounts/abi/argument.go
View file

@ -92,9 +92,8 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error {
if len(data) == 0 {
if len(arguments) != 0 {
return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
} else {
return nil // Nothing to unmarshal, return
}
return nil // Nothing to unmarshal, return
}
// make sure the passed value is arguments pointer
if reflect.Ptr != reflect.ValueOf(v).Kind() {
@ -104,6 +103,9 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error {
if err != nil {
return err
}
if len(marshalledValues) == 0 {
return fmt.Errorf("abi: Unpack(no-values unmarshalled %T)", v)
}
if arguments.isTuple() {
return arguments.unpackTuple(v, marshalledValues)
}
@ -112,18 +114,24 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error {
// UnpackIntoMap performs the operation hexdata -> mapping of argument name to argument value
func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) error {
// Make sure map is not nil
if v == nil {
return fmt.Errorf("abi: cannot unpack into a nil map")
}
if len(data) == 0 {
if len(arguments) != 0 {
return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
} else {
return nil // Nothing to unmarshal, return
}
return nil // Nothing to unmarshal, return
}
marshalledValues, err := arguments.UnpackValues(data)
if err != nil {
return err
}
return arguments.unpackIntoMap(v, marshalledValues)
for i, arg := range arguments.NonIndexed() {
v[arg.Name] = marshalledValues[i]
}
return nil
}
// unpack sets the unmarshalled value to go format.
@ -195,19 +203,6 @@ func unpack(t *Type, dst interface{}, src interface{}) error {
return nil
}
// unpackIntoMap unpacks marshalledValues into the provided map[string]interface{}
func (arguments Arguments) unpackIntoMap(v map[string]interface{}, marshalledValues []interface{}) error {
// Make sure map is not nil
if v == nil {
return fmt.Errorf("abi: cannot unpack into a nil map")
}
for i, arg := range arguments.NonIndexed() {
v[arg.Name] = marshalledValues[i]
}
return nil
}
// unpackAtomic unpacks ( hexdata -> go ) a single value
func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interface{}) error {
if arguments.LengthNonIndexed() == 0 {
@ -233,30 +228,28 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interfac
// unpackTuple unpacks ( hexdata -> go ) a batch of values.
func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error {
var (
value = reflect.ValueOf(v).Elem()
typ = value.Type()
kind = value.Kind()
value = reflect.ValueOf(v).Elem()
typ = value.Type()
kind = value.Kind()
nonIndexedArgs = arguments.NonIndexed()
)
if err := requireUnpackKind(value, typ, kind, arguments); err != nil {
if err := requireUnpackKind(value, len(nonIndexedArgs), arguments); err != nil {
return err
}
// If the interface is a struct, get of abi->struct_field mapping
var abi2struct map[string]string
if kind == reflect.Struct {
var (
argNames []string
err error
)
for _, arg := range arguments.NonIndexed() {
argNames = append(argNames, arg.Name)
argNames := make([]string, len(nonIndexedArgs))
for i, arg := range nonIndexedArgs {
argNames[i] = arg.Name
}
abi2struct, err = mapArgNamesToStructFields(argNames, value)
if err != nil {
var err error
if abi2struct, err = mapArgNamesToStructFields(argNames, value); err != nil {
return err
}
}
for i, arg := range arguments.NonIndexed() {
for i, arg := range nonIndexedArgs {
switch kind {
case reflect.Struct:
field := value.FieldByName(abi2struct[arg.Name])
@ -292,7 +285,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
retval := make([]interface{}, 0, arguments.LengthNonIndexed())
virtualArgs := 0
for index, arg := range arguments.NonIndexed() {
marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
marshalledValue, err := ToGoType((index+virtualArgs)*32, arg.Type, data)
if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) {
// If we have a static array, like [3]uint256, these are coded as
// just like uint256,uint256,uint256.

46
accounts/abi/bind/backends/simulated.go
View file

@ -124,7 +124,19 @@ func (b *SimulatedBackend) rollback() {
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
}
// stateByBlockNumber retrieves a state by a given blocknumber.
func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *big.Int) (*state.StateDB, error) {
if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) == 0 {
return b.blockchain.State()
}
block, err := b.BlockByNumber(ctx, blockNumber)
if err != nil {
return nil, err
}
return b.blockchain.StateAt(block.Hash())
}
// CodeAt returns the code associated with a certain account in the blockchain.
@ -132,10 +144,11 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address,
b.mu.Lock()
defer b.mu.Unlock()
if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 {
return nil, errBlockNumberUnsupported
statedb, err := b.stateByBlockNumber(ctx, blockNumber)
if err != nil {
return nil, err
}
statedb, _ := b.blockchain.State()
return statedb.GetCode(contract), nil
}
@ -144,10 +157,11 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres
b.mu.Lock()
defer b.mu.Unlock()
if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 {
return nil, errBlockNumberUnsupported
statedb, err := b.stateByBlockNumber(ctx, blockNumber)
if err != nil {
return nil, err
}
statedb, _ := b.blockchain.State()
return statedb.GetBalance(contract), nil
}
@ -156,10 +170,11 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address,
b.mu.Lock()
defer b.mu.Unlock()
if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 {
return 0, errBlockNumberUnsupported
statedb, err := b.stateByBlockNumber(ctx, blockNumber)
if err != nil {
return 0, err
}
statedb, _ := b.blockchain.State()
return statedb.GetNonce(contract), nil
}
@ -168,10 +183,11 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
b.mu.Lock()
defer b.mu.Unlock()
if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 {
return nil, errBlockNumberUnsupported
statedb, err := b.stateByBlockNumber(ctx, blockNumber)
if err != nil {
return nil, err
}
statedb, _ := b.blockchain.State()
val := statedb.GetState(contract, key)
return val[:], nil
}
@ -464,7 +480,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
return nil
}
@ -577,7 +593,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
return nil
}

16
accounts/abi/bind/base.go
View file

@ -171,12 +171,24 @@ func (c *BoundContract) Transact(opts *TransactOpts, method string, params ...in
if err != nil {
return nil, err
}
// todo(rjl493456442) check the method is payable or not,
// reject invalid transaction at the first place
return c.transact(opts, &c.address, input)
}
// RawTransact initiates a transaction with the given raw calldata as the input.
// It's usually used to initiates transaction for invoking **Fallback** function.
func (c *BoundContract) RawTransact(opts *TransactOpts, calldata []byte) (*types.Transaction, error) {
// todo(rjl493456442) check the method is payable or not,
// reject invalid transaction at the first place
return c.transact(opts, &c.address, calldata)
}
// Transfer initiates a plain transaction to move funds to the contract, calling
// its default method if one is available.
func (c *BoundContract) Transfer(opts *TransactOpts) (*types.Transaction, error) {
// todo(rjl493456442) check the payable fallback or receive is defined
// or not, reject invalid transaction at the first place
return c.transact(opts, &c.address, nil)
}
@ -252,7 +264,7 @@ func (c *BoundContract) FilterLogs(opts *FilterOpts, name string, query ...[]int
opts = new(FilterOpts)
}
// Append the event selector to the query parameters and construct the topic set
query = append([][]interface{}{{c.abi.Events[name].ID()}}, query...)
query = append([][]interface{}{{c.abi.Events[name].ID}}, query...)
topics, err := makeTopics(query...)
if err != nil {
@ -301,7 +313,7 @@ func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]inter
opts = new(WatchOpts)
}
// Append the event selector to the query parameters and construct the topic set
query = append([][]interface{}{{c.abi.Events[name].ID()}}, query...)
query = append([][]interface{}{{c.abi.Events[name].ID}}, query...)
topics, err := makeTopics(query...)
if err != nil {

35
accounts/abi/bind/bind.go
View file

@ -77,6 +77,8 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
calls = make(map[string]*tmplMethod)
transacts = make(map[string]*tmplMethod)
events = make(map[string]*tmplEvent)
fallback *tmplMethod
receive *tmplMethod
// identifiers are used to detect duplicated identifier of function
// and event. For all calls, transacts and events, abigen will generate
@ -92,7 +94,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
normalizedName := methodNormalizer[lang](alias(aliases, original.Name))
// Ensure there is no duplicated identifier
var identifiers = callIdentifiers
if !original.Const {
if !original.IsConstant() {
identifiers = transactIdentifiers
}
if identifiers[normalizedName] {
@ -121,7 +123,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
}
}
// Append the methods to the call or transact lists
if original.Const {
if original.IsConstant() {
calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
} else {
transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
@ -156,7 +158,13 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
// Append the event to the accumulator list
events[original.Name] = &tmplEvent{Original: original, Normalized: normalized}
}
// Add two special fallback functions if they exist
if evmABI.HasFallback() {
fallback = &tmplMethod{Original: evmABI.Fallback}
}
if evmABI.HasReceive() {
receive = &tmplMethod{Original: evmABI.Receive}
}
// There is no easy way to pass arbitrary java objects to the Go side.
if len(structs) > 0 && lang == LangJava {
return "", errors.New("java binding for tuple arguments is not supported yet")
@ -169,6 +177,8 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
Constructor: evmABI.Constructor,
Calls: calls,
Transacts: transacts,
Fallback: fallback,
Receive: receive,
Events: events,
Libraries: make(map[string]string),
}
@ -619,11 +629,22 @@ func formatMethod(method abi.Method, structs map[string]*tmplStruct) string {
outputs[i] += fmt.Sprintf(" %v", output.Name)
}
}
constant := ""
if method.Const {
constant = "constant "
// Extract meaningful state mutability of solidity method.
// If it's default value, never print it.
state := method.StateMutability
if state == "nonpayable" {
state = ""
}
return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
if state != "" {
state = state + " "
}
identity := fmt.Sprintf("function %v", method.RawName)
if method.Type == abi.Fallback {
identity = "fallback"
} else if method.Type == abi.Receive {
identity = "receive"
}
return fmt.Sprintf("%s(%v) %sreturns(%v)", identity, strings.Join(inputs, ", "), state, strings.Join(outputs, ", "))
}
// formatEvent transforms raw event representation into a user friendly one.

134
accounts/abi/bind/bind_test.go
View file

File diff suppressed because one or more lines are too long

66
accounts/abi/bind/template.go
View file

@ -35,6 +35,8 @@ type tmplContract struct {
Constructor abi.Method // Contract constructor for deploy parametrization
Calls map[string]*tmplMethod // Contract calls that only read state data
Transacts map[string]*tmplMethod // Contract calls that write state data
Fallback *tmplMethod // Additional special fallback function
Receive *tmplMethod // Additional special receive function
Events map[string]*tmplEvent // Contract events accessors
Libraries map[string]string // Same as tmplData, but filtered to only keep what the contract needs
Library bool // Indicator whether the contract is a library
@ -351,6 +353,52 @@ var (
}
{{end}}
{{if .Fallback}}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Transactor) Fallback(opts *bind.TransactOpts, calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.contract.RawTransact(opts, calldata)
}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Session) Fallback(calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Fallback(&_{{$contract.Type}}.TransactOpts, calldata)
}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) Fallback(calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Fallback(&_{{$contract.Type}}.TransactOpts, calldata)
}
{{end}}
{{if .Receive}}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Transactor) Receive(opts *bind.TransactOpts) (*types.Transaction, error) {
return _{{$contract.Type}}.contract.RawTransact(opts, nil) // calldata is disallowed for receive function
}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Session) Receive() (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Receive(&_{{$contract.Type}}.TransactOpts)
}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) Receive() (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Receive(&_{{$contract.Type}}.TransactOpts)
}
{{end}}
{{range .Events}}
// {{$contract.Type}}{{.Normalized.Name}}Iterator is returned from Filter{{.Normalized.Name}} and is used to iterate over the raw logs and unpacked data for {{.Normalized.Name}} events raised by the {{$contract.Type}} contract.
type {{$contract.Type}}{{.Normalized.Name}}Iterator struct {
@ -611,6 +659,24 @@ import java.util.*;
return this.Contract.transact(opts, "{{.Original.Name}}" , args);
}
{{end}}
{{if .Fallback}}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
public Transaction Fallback(TransactOpts opts, byte[] calldata) throws Exception {
return this.Contract.rawTransact(opts, calldata);
}
{{end}}
{{if .Receive}}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
public Transaction Receive(TransactOpts opts) throws Exception {
return this.Contract.rawTransact(opts, null);
}
{{end}}
}
{{end}}
`

180
accounts/abi/bind/topics.go
View file

@ -49,17 +49,13 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
topic[common.HashLength-1] = 1
}
case int8:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 1))
case int16:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 2))
case int32:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 4))
case int64:
blob := big.NewInt(rule).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(rule, 8))
case uint8:
blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
@ -103,151 +99,75 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
return topics, nil
}
// Big batch of reflect types for topic reconstruction.
var (
reflectHash = reflect.TypeOf(common.Hash{})
reflectAddress = reflect.TypeOf(common.Address{})
reflectBigInt = reflect.TypeOf(new(big.Int))
)
func genIntType(rule int64, size uint) []byte {
var topic [common.HashLength]byte
if rule < 0 {
// if a rule is negative, we need to put it into two's complement.
// extended to common.Hashlength bytes.
topic = [common.HashLength]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}
}
for i := uint(0); i < size; i++ {
topic[common.HashLength-i-1] = byte(rule >> (i * 8))
}
return topic[:]
}
// parseTopics converts the indexed topic fields into actual log field values.
//
// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
// hashes as the topic value!
func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) error {
// Sanity check that the fields and topics match up
if len(fields) != len(topics) {
return errors.New("topic/field count mismatch")
}
// Iterate over all the fields and reconstruct them from topics
for _, arg := range fields {
if !arg.Indexed {
return errors.New("non-indexed field in topic reconstruction")
}
field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
// Try to parse the topic back into the fields based on primitive types
switch field.Kind() {
case reflect.Bool:
if topics[0][common.HashLength-1] == 1 {
field.Set(reflect.ValueOf(true))
}
case reflect.Int8:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int8(num.Int64())))
case reflect.Int16:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int16(num.Int64())))
case reflect.Int32:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int32(num.Int64())))
case reflect.Int64:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(num.Int64()))
case reflect.Uint8:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint8(num.Uint64())))
case reflect.Uint16:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint16(num.Uint64())))
case reflect.Uint32:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint32(num.Uint64())))
case reflect.Uint64:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(num.Uint64()))
default:
// Ran out of plain primitive types, try custom types
switch field.Type() {
case reflectHash: // Also covers all dynamic types
field.Set(reflect.ValueOf(topics[0]))
case reflectAddress:
var addr common.Address
copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
field.Set(reflect.ValueOf(addr))
case reflectBigInt:
num := new(big.Int).SetBytes(topics[0][:])
if arg.Type.T == abi.IntTy {
if num.Cmp(abi.MaxInt256) > 0 {
num.Add(abi.MaxUint256, big.NewInt(0).Neg(num))
num.Add(num, big.NewInt(1))
num.Neg(num)
}
}
field.Set(reflect.ValueOf(num))
default:
// Ran out of custom types, try the crazies
switch {
// static byte array
case arg.Type.T == abi.FixedBytesTy:
reflect.Copy(field, reflect.ValueOf(topics[0][:arg.Type.Size]))
default:
return fmt.Errorf("unsupported indexed type: %v", arg.Type)
}
}
}
topics = topics[1:]
}
return nil
return parseTopicWithSetter(fields, topics,
func(arg abi.Argument, reconstr interface{}) {
field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
field.Set(reflect.ValueOf(reconstr))
})
}
// parseTopicsIntoMap converts the indexed topic field-value pairs into map key-value pairs
func parseTopicsIntoMap(out map[string]interface{}, fields abi.Arguments, topics []common.Hash) error {
return parseTopicWithSetter(fields, topics,
func(arg abi.Argument, reconstr interface{}) {
out[arg.Name] = reconstr
})
}
// parseTopicWithSetter converts the indexed topic field-value pairs and stores them using the
// provided set function.
//
// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
// hashes as the topic value!
func parseTopicWithSetter(fields abi.Arguments, topics []common.Hash, setter func(abi.Argument, interface{})) error {
// Sanity check that the fields and topics match up
if len(fields) != len(topics) {
return errors.New("topic/field count mismatch")
}
// Iterate over all the fields and reconstruct them from topics
for _, arg := range fields {
for i, arg := range fields {
if !arg.Indexed {
return errors.New("non-indexed field in topic reconstruction")
}
var reconstr interface{}
switch arg.Type.T {
case abi.BoolTy:
out[arg.Name] = topics[0][common.HashLength-1] == 1
case abi.IntTy, abi.UintTy:
out[arg.Name] = abi.ReadInteger(arg.Type.T, arg.Type.Kind, topics[0].Bytes())
case abi.AddressTy:
var addr common.Address
copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
out[arg.Name] = addr
case abi.HashTy:
out[arg.Name] = topics[0]
case abi.FixedBytesTy:
array, err := abi.ReadFixedBytes(arg.Type, topics[0].Bytes())
if err != nil {
return err
}
out[arg.Name] = array
case abi.TupleTy:
return errors.New("tuple type in topic reconstruction")
case abi.StringTy, abi.BytesTy, abi.SliceTy, abi.ArrayTy:
// Array types (including strings and bytes) have their keccak256 hashes stored in the topic- not a hash
// whose bytes can be decoded to the actual value- so the best we can do is retrieve that hash
out[arg.Name] = topics[0]
reconstr = topics[i]
case abi.FunctionTy:
if garbage := binary.BigEndian.Uint64(topics[0][0:8]); garbage != 0 {
return fmt.Errorf("bind: got improperly encoded function type, got %v", topics[0].Bytes())
if garbage := binary.BigEndian.Uint64(topics[i][0:8]); garbage != 0 {
return fmt.Errorf("bind: got improperly encoded function type, got %v", topics[i].Bytes())
}
var tmp [24]byte
copy(tmp[:], topics[0][8:32])
out[arg.Name] = tmp
default: // Not handling tuples
return fmt.Errorf("unsupported indexed type: %v", arg.Type)
copy(tmp[:], topics[i][8:32])
reconstr = tmp
default:
var err error
reconstr, err = abi.ToGoType(0, arg.Type, topics[i].Bytes())
if err != nil {
return err
}
}
topics = topics[1:]
// Use the setter function to store the value
setter(arg, reconstr)
}
return nil

91
accounts/abi/bind/topics_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
)
func TestMakeTopics(t *testing.T) {
@ -41,6 +42,80 @@ func TestMakeTopics(t *testing.T) {
[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
false,
},
{
"support common hash types in topics",
args{[][]interface{}{{common.Hash{1, 2, 3, 4, 5}}}},
[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
false,
},
{
"support address types in topics",
args{[][]interface{}{{common.Address{1, 2, 3, 4, 5}}}},
[][]common.Hash{{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5}}},
false,
},
{
"support *big.Int types in topics",
args{[][]interface{}{{big.NewInt(1).Lsh(big.NewInt(2), 254)}}},
[][]common.Hash{{common.Hash{128}}},
false,
},
{
"support boolean types in topics",
args{[][]interface{}{
{true},
{false},
}},
[][]common.Hash{
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0}},
},
false,
},
{
"support int/uint(8/16/32/64) types in topics",
args{[][]interface{}{
{int8(-2)},
{int16(-3)},
{int32(-4)},
{int64(-5)},
{int8(1)},
{int16(256)},
{int32(65536)},
{int64(4294967296)},
{uint8(1)},
{uint16(256)},
{uint32(65536)},
{uint64(4294967296)},
}},
[][]common.Hash{
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 254}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 253}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 252}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 251}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0}},
},
false,
},
{
"support string types in topics",
args{[][]interface{}{{"hello world"}}},
[][]common.Hash{{crypto.Keccak256Hash([]byte("hello world"))}},
false,
},
{
"support byte slice types in topics",
args{[][]interface{}{{[]byte{1, 2, 3}}}},
[][]common.Hash{{crypto.Keccak256Hash([]byte{1, 2, 3})}},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -84,6 +159,7 @@ func setupTopicsTests() []topicTest {
bytesType, _ := abi.NewType("bytes5", "", nil)
int8Type, _ := abi.NewType("int8", "", nil)
int256Type, _ := abi.NewType("int256", "", nil)
tupleType, _ := abi.NewType("tuple(int256,int8)", "", nil)
tests := []topicTest{
{
@ -145,6 +221,21 @@ func setupTopicsTests() []topicTest {
},
wantErr: false,
},
{
name: "tuple(int256, int8)",
args: args{
createObj: func() interface{} { return nil },
resultObj: func() interface{} { return nil },
resultMap: func() map[string]interface{} { return make(map[string]interface{}) },
fields: abi.Arguments{abi.Argument{
Name: "tupletype",
Type: tupleType,
Indexed: true,
}},
topics: []common.Hash{},
},
wantErr: true,
},
}
return tests

81
accounts/abi/event.go
View file

@ -42,36 +42,59 @@ type Event struct {
RawName string
Anonymous bool
Inputs Arguments
str string
// Sig contains the string signature according to the ABI spec.
// e.g. event foo(uint32 a, int b) = "foo(uint32,int256)"
// Please note that "int" is substitute for its canonical representation "int256"
Sig string
// ID returns the canonical representation of the event's signature used by the
// abi definition to identify event names and types.
ID common.Hash
}
// NewEvent creates a new Event.
// It sanitizes the input arguments to remove unnamed arguments.
// It also precomputes the id, signature and string representation
// of the event.
func NewEvent(name, rawName string, anonymous bool, inputs Arguments) Event {
// sanitize inputs to remove inputs without names
// and precompute string and sig representation.
names := make([]string, len(inputs))
types := make([]string, len(inputs))
for i, input := range inputs {
if input.Name == "" {
inputs[i] = Argument{
Name: fmt.Sprintf("arg%d", i),
Indexed: input.Indexed,
Type: input.Type,
}
} else {
inputs[i] = input
}
// string representation
names[i] = fmt.Sprintf("%v %v", input.Type, inputs[i].Name)
if input.Indexed {
names[i] = fmt.Sprintf("%v indexed %v", input.Type, inputs[i].Name)
}
// sig representation
types[i] = input.Type.String()
}
str := fmt.Sprintf("event %v(%v)", rawName, strings.Join(names, ", "))
sig := fmt.Sprintf("%v(%v)", rawName, strings.Join(types, ","))
id := common.BytesToHash(crypto.Keccak256([]byte(sig)))
return Event{
Name: name,
RawName: rawName,
Anonymous: anonymous,
Inputs: inputs,
str: str,
Sig: sig,
ID: id,
}
}
func (e Event) String() string {
inputs := make([]string, len(e.Inputs))
for i, input := range e.Inputs {
inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name)
if input.Indexed {
inputs[i] = fmt.Sprintf("%v indexed %v", input.Type, input.Name)
}
}
return fmt.Sprintf("event %v(%v)", e.RawName, strings.Join(inputs, ", "))
}
// Sig returns the event string signature according to the ABI spec.
//
// Example
//
// event foo(uint32 a, int b) = "foo(uint32,int256)"
//
// Please note that "int" is substitute for its canonical representation "int256"
func (e Event) Sig() string {
types := make([]string, len(e.Inputs))
for i, input := range e.Inputs {
types[i] = input.Type.String()
}
return fmt.Sprintf("%v(%v)", e.RawName, strings.Join(types, ","))
}
// ID returns the canonical representation of the event's signature used by the
// abi definition to identify event names and types.
func (e Event) ID() common.Hash {
return common.BytesToHash(crypto.Keccak256([]byte(e.Sig())))
return e.str
}

4
accounts/abi/event_test.go
View file

@ -104,8 +104,8 @@ func TestEventId(t *testing.T) {
}
for name, event := range abi.Events {
if event.ID() != test.expectations[name] {
t.Errorf("expected id to be %x, got %x", test.expectations[name], event.ID())
if event.ID != test.expectations[name] {
t.Errorf("expected id to be %x, got %x", test.expectations[name], event.ID)
}
}
}

147
accounts/abi/method.go
View file

@ -23,6 +23,24 @@ import (
"github.com/ethereum/go-ethereum/crypto"
)
// FunctionType represents different types of functions a contract might have.
type FunctionType int
const (
// Constructor represents the constructor of the contract.
// The constructor function is called while deploying a contract.
Constructor FunctionType = iota
// Fallback represents the fallback function.
// This function is executed if no other function matches the given function
// signature and no receive function is specified.
Fallback
// Receive represents the receive function.
// This function is executed on plain Ether transfers.
Receive
// Function represents a normal function.
Function
)
// Method represents a callable given a `Name` and whether the method is a constant.
// If the method is `Const` no transaction needs to be created for this
// particular Method call. It can easily be simulated using a local VM.
@ -41,50 +59,109 @@ type Method struct {
// * foo(uint,uint)
// The method name of the first one will be resolved as foo while the second one
// will be resolved as foo0.
Name string
// RawName is the raw method name parsed from ABI.
RawName string
Const bool
Name string
RawName string // RawName is the raw method name parsed from ABI
// Type indicates whether the method is a
// special fallback introduced in solidity v0.6.0
Type FunctionType
// StateMutability indicates the mutability state of method,
// the default value is nonpayable. It can be empty if the abi
// is generated by legacy compiler.
StateMutability string
// Legacy indicators generated by compiler before v0.6.0
Constant bool
Payable bool
Inputs Arguments
Outputs Arguments
str string
// Sig returns the methods string signature according to the ABI spec.
// e.g. function foo(uint32 a, int b) = "foo(uint32,int256)"
// Please note that "int" is substitute for its canonical representation "int256"
Sig string
// ID returns the canonical representation of the method's signature used by the
// abi definition to identify method names and types.
ID []byte
}
// Sig returns the methods string signature according to the ABI spec.
//
// Example
//
// function foo(uint32 a, int b) = "foo(uint32,int256)"
//
// Please note that "int" is substitute for its canonical representation "int256"
func (method Method) Sig() string {
types := make([]string, len(method.Inputs))
for i, input := range method.Inputs {
// NewMethod creates a new Method.
// A method should always be created using NewMethod.
// It also precomputes the sig representation and the string representation
// of the method.
func NewMethod(name string, rawName string, funType FunctionType, mutability string, isConst, isPayable bool, inputs Arguments, outputs Arguments) Method {
var (
types = make([]string, len(inputs))
inputNames = make([]string, len(inputs))
outputNames = make([]string, len(outputs))
)
for i, input := range inputs {
inputNames[i] = fmt.Sprintf("%v %v", input.Type, input.Name)
types[i] = input.Type.String()
}
return fmt.Sprintf("%v(%v)", method.RawName, strings.Join(types, ","))
for i, output := range outputs {
outputNames[i] = output.Type.String()
if len(output.Name) > 0 {
outputNames[i] += fmt.Sprintf(" %v", output.Name)
}
}
// calculate the signature and method id. Note only function
// has meaningful signature and id.
var (
sig string
id []byte
)
if funType == Function {
sig = fmt.Sprintf("%v(%v)", rawName, strings.Join(types, ","))
id = crypto.Keccak256([]byte(sig))[:4]
}
// Extract meaningful state mutability of solidity method.
// If it's default value, never print it.
state := mutability
if state == "nonpayable" {
state = ""
}
if state != "" {
state = state + " "
}
identity := fmt.Sprintf("function %v", rawName)
if funType == Fallback {
identity = "fallback"
} else if funType == Receive {
identity = "receive"
} else if funType == Constructor {
identity = "constructor"
}
str := fmt.Sprintf("%v(%v) %sreturns(%v)", identity, strings.Join(inputNames, ", "), state, strings.Join(outputNames, ", "))
return Method{
Name: name,
RawName: rawName,
Type: funType,
StateMutability: mutability,
Constant: isConst,
Payable: isPayable,
Inputs: inputs,
Outputs: outputs,
str: str,
Sig: sig,
ID: id,
}
}
func (method Method) String() string {
inputs := make([]string, len(method.Inputs))
for i, input := range method.Inputs {
inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name)
}
outputs := make([]string, len(method.Outputs))
for i, output := range method.Outputs {
outputs[i] = output.Type.String()
if len(output.Name) > 0 {
outputs[i] += fmt.Sprintf(" %v", output.Name)
}
}
constant := ""
if method.Const {
constant = "constant "
}
return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
return method.str
}
// ID returns the canonical representation of the method's signature used by the
// abi definition to identify method names and types.
func (method Method) ID() []byte {
return crypto.Keccak256([]byte(method.Sig()))[:4]
// IsConstant returns the indicator whether the method is read-only.
func (method Method) IsConstant() bool {
return method.StateMutability == "view" || method.StateMutability == "pure" || method.Constant
}
// IsPayable returns the indicator whether the method can process
// plain ether transfers.
func (method Method) IsPayable() bool {
return method.StateMutability == "payable" || method.Payable
}

31
accounts/abi/method_test.go
View file

@ -23,13 +23,15 @@ import (
const methoddata = `
[
{"type": "function", "name": "balance", "constant": true },
{"type": "function", "name": "send", "constant": false, "inputs": [{ "name": "amount", "type": "uint256" }]},
{"type": "function", "name": "transfer", "constant": false, "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]},
{"type": "function", "name": "balance", "stateMutability": "view"},
{"type": "function", "name": "send", "inputs": [{ "name": "amount", "type": "uint256" }]},
{"type": "function", "name": "transfer", "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple"}],"name":"tuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[]"}],"name":"tupleSlice","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5]"}],"name":"tupleArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"stateMutability":"nonpayable","type":"fallback"},
{"stateMutability":"payable","type":"receive"}
]`
func TestMethodString(t *testing.T) {
@ -39,7 +41,7 @@ func TestMethodString(t *testing.T) {
}{
{
method: "balance",
expectation: "function balance() constant returns()",
expectation: "function balance() view returns()",
},
{
method: "send",
@ -65,6 +67,14 @@ func TestMethodString(t *testing.T) {
method: "complexTuple",
expectation: "function complexTuple((uint256,uint256)[5][] a) returns()",
},
{
method: "fallback",
expectation: "fallback() returns()",
},
{
method: "receive",
expectation: "receive() payable returns()",
},
}
abi, err := JSON(strings.NewReader(methoddata))
@ -73,7 +83,14 @@ func TestMethodString(t *testing.T) {
}
for _, test := range table {
got := abi.Methods[test.method].String()
var got string
if test.method == "fallback" {
got = abi.Fallback.String()
} else if test.method == "receive" {
got = abi.Receive.String()
} else {
got = abi.Methods[test.method].String()
}
if got != test.expectation {
t.Errorf("expected string to be %s, got %s", test.expectation, got)
}
@ -120,7 +137,7 @@ func TestMethodSig(t *testing.T) {
}
for _, test := range cases {
got := abi.Methods[test.method].Sig()
got := abi.Methods[test.method].Sig
if got != test.expect {
t.Errorf("expected string to be %s, got %s", test.expect, got)
}

14
accounts/abi/pack_test.go
View file

@ -634,7 +634,7 @@ func TestMethodPack(t *testing.T) {
t.Fatal(err)
}
sig := abi.Methods["slice"].ID()
sig := abi.Methods["slice"].ID
sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@ -648,7 +648,7 @@ func TestMethodPack(t *testing.T) {
}
var addrA, addrB = common.Address{1}, common.Address{2}
sig = abi.Methods["sliceAddress"].ID()
sig = abi.Methods["sliceAddress"].ID
sig = append(sig, common.LeftPadBytes([]byte{32}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
sig = append(sig, common.LeftPadBytes(addrA[:], 32)...)
@ -663,7 +663,7 @@ func TestMethodPack(t *testing.T) {
}
var addrC, addrD = common.Address{3}, common.Address{4}
sig = abi.Methods["sliceMultiAddress"].ID()
sig = abi.Methods["sliceMultiAddress"].ID
sig = append(sig, common.LeftPadBytes([]byte{64}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{160}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@ -681,7 +681,7 @@ func TestMethodPack(t *testing.T) {
t.Errorf("expected %x got %x", sig, packed)
}
sig = abi.Methods["slice256"].ID()
sig = abi.Methods["slice256"].ID
sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@ -695,7 +695,7 @@ func TestMethodPack(t *testing.T) {
}
a := [2][2]*big.Int{{big.NewInt(1), big.NewInt(1)}, {big.NewInt(2), big.NewInt(0)}}
sig = abi.Methods["nestedArray"].ID()
sig = abi.Methods["nestedArray"].ID
sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@ -712,7 +712,7 @@ func TestMethodPack(t *testing.T) {
t.Errorf("expected %x got %x", sig, packed)
}
sig = abi.Methods["nestedArray2"].ID()
sig = abi.Methods["nestedArray2"].ID
sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{0x80}, 32)...)
@ -728,7 +728,7 @@ func TestMethodPack(t *testing.T) {
t.Errorf("expected %x got %x", sig, packed)
}
sig = abi.Methods["nestedSlice"].ID()
sig = abi.Methods["nestedSlice"].ID
sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{0x02}, 32)...)
sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)

17
accounts/abi/reflect.go
View file

@ -118,18 +118,16 @@ func requireAssignable(dst, src reflect.Value) error {
}
// requireUnpackKind verifies preconditions for unpacking `args` into `kind`
func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind,
args Arguments) error {
switch k {
func requireUnpackKind(v reflect.Value, minLength int, args Arguments) error {
switch v.Kind() {
case reflect.Struct:
case reflect.Slice, reflect.Array:
if minLen := args.LengthNonIndexed(); v.Len() < minLen {
if v.Len() < minLength {
return fmt.Errorf("abi: insufficient number of elements in the list/array for unpack, want %d, got %d",
minLen, v.Len())
minLength, v.Len())
}
default:
return fmt.Errorf("abi: cannot unmarshal tuple into %v", t)
return fmt.Errorf("abi: cannot unmarshal tuple into %v", v.Type())
}
return nil
}
@ -156,9 +154,8 @@ func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[stri
continue
}
// skip fields that have no abi:"" tag.
var ok bool
var tagName string
if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok {
tagName, ok := typ.Field(i).Tag.Lookup("abi")
if !ok {
continue
}
// check if tag is empty.

30
accounts/abi/unpack.go
View file

@ -27,13 +27,9 @@ import (
var (
// MaxUint256 is the maximum value that can be represented by a uint256
MaxUint256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
big.NewInt(-1))
MaxUint256 = new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 256), common.Big1)
// MaxInt256 is the maximum value that can be represented by a int256
MaxInt256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
big.NewInt(-1))
MaxInt256 = new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 255), common.Big1)
)
// ReadInteger reads the integer based on its kind and returns the appropriate value
@ -56,17 +52,17 @@ func ReadInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
case reflect.Int64:
return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
default:
// the only case lefts for integer is int256/uint256.
// big.SetBytes can't tell if a number is negative, positive on itself.
// On EVM, if the returned number > max int256, it is negative.
// the only case left for integer is int256/uint256.
ret := new(big.Int).SetBytes(b)
if typ == UintTy {
return ret
}
if ret.Cmp(MaxInt256) > 0 {
ret.Add(MaxUint256, big.NewInt(0).Neg(ret))
ret.Add(ret, big.NewInt(1))
// big.SetBytes can't tell if a number is negative or positive in itself.
// On EVM, if the returned number > max int256, it is negative.
// A number is > max int256 if the bit at position 255 is set.
if ret.Bit(255) == 1 {
ret.Add(MaxUint256, new(big.Int).Neg(ret))
ret.Add(ret, common.Big1)
ret.Neg(ret)
}
return ret
@ -144,7 +140,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
elemSize := getTypeSize(*t.Elem)
for i, j := start, 0; j < size; i, j = i+elemSize, j+1 {
inter, err := toGoType(i, *t.Elem, output)
inter, err := ToGoType(i, *t.Elem, output)
if err != nil {
return nil, err
}
@ -161,7 +157,7 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
retval := reflect.New(t.Type).Elem()
virtualArgs := 0
for index, elem := range t.TupleElems {
marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output)
marshalledValue, err := ToGoType((index+virtualArgs)*32, *elem, output)
if elem.T == ArrayTy && !isDynamicType(*elem) {
// If we have a static array, like [3]uint256, these are coded as
// just like uint256,uint256,uint256.
@ -187,9 +183,9 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
return retval.Interface(), nil
}
// toGoType parses the output bytes and recursively assigns the value of these bytes
// ToGoType parses the output bytes and recursively assigns the value of these bytes
// into a go type with accordance with the ABI spec.
func toGoType(index int, t Type, output []byte) (interface{}, error) {
func ToGoType(index int, t Type, output []byte) (interface{}, error) {
if index+32 > len(output) {
return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32)
}

38
accounts/abi/unpack_test.go
View file

@ -443,7 +443,7 @@ var unpackTests = []unpackTest{
func TestUnpack(t *testing.T) {
for i, test := range unpackTests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
def := fmt.Sprintf(`[{ "name" : "method", "type": "function", "outputs": %s}]`, test.def)
abi, err := JSON(strings.NewReader(def))
if err != nil {
t.Fatalf("invalid ABI definition %s: %v", def, err)
@ -522,7 +522,7 @@ type methodMultiOutput struct {
func methodMultiReturn(require *require.Assertions) (ABI, []byte, methodMultiOutput) {
const definition = `[
{ "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
{ "name" : "multi", "type": "function", "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
var expected = methodMultiOutput{big.NewInt(1), "hello"}
abi, err := JSON(strings.NewReader(definition))
@ -611,7 +611,7 @@ func TestMethodMultiReturn(t *testing.T) {
}
func TestMultiReturnWithArray(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -634,7 +634,7 @@ func TestMultiReturnWithArray(t *testing.T) {
}
func TestMultiReturnWithStringArray(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -664,7 +664,7 @@ func TestMultiReturnWithStringArray(t *testing.T) {
}
func TestMultiReturnWithStringSlice(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -700,7 +700,7 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
// values of nested static arrays count towards the size as well, and any element following
// after such nested array argument should be read with the correct offset,
// so that it does not read content from the previous array argument.
const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -737,15 +737,15 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
func TestUnmarshal(t *testing.T) {
const definition = `[
{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },
{ "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] },
{ "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] },
{ "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] },
{ "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] },
{ "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] },
{ "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] },
{ "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] },
{ "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]`
{ "name" : "int", "type": "function", "outputs": [ { "type": "uint256" } ] },
{ "name" : "bool", "type": "function", "outputs": [ { "type": "bool" } ] },
{ "name" : "bytes", "type": "function", "outputs": [ { "type": "bytes" } ] },
{ "name" : "fixed", "type": "function", "outputs": [ { "type": "bytes32" } ] },
{ "name" : "multi", "type": "function", "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] },
{ "name" : "intArraySingle", "type": "function", "outputs": [ { "type": "uint256[3]" } ] },
{ "name" : "addressSliceSingle", "type": "function", "outputs": [ { "type": "address[]" } ] },
{ "name" : "addressSliceDouble", "type": "function", "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] },
{ "name" : "mixedBytes", "type": "function", "stateMutability" : "view", "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
@ -985,7 +985,7 @@ func TestUnmarshal(t *testing.T) {
}
func TestUnpackTuple(t *testing.T) {
const simpleTuple = `[{"name":"tuple","constant":false,"outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]`
const simpleTuple = `[{"name":"tuple","type":"function","outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]`
abi, err := JSON(strings.NewReader(simpleTuple))
if err != nil {
t.Fatal(err)
@ -1009,12 +1009,12 @@ func TestUnpackTuple(t *testing.T) {
t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.A)
}
if v.B.Cmp(big.NewInt(-1)) != 0 {
t.Errorf("unexpected value unpacked: want %x, got %x", v.B, -1)
t.Errorf("unexpected value unpacked: want %x, got %x", -1, v.B)
}
}
// Test nested tuple
const nestedTuple = `[{"name":"tuple","constant":false,"outputs":[
const nestedTuple = `[{"name":"tuple","type":"function","outputs":[
{"type":"tuple","name":"s","components":[{"type":"uint256","name":"a"},{"type":"uint256[]","name":"b"},{"type":"tuple[]","name":"c","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]}]},
{"type":"tuple","name":"t","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]},
{"type":"uint256","name":"a"}
@ -1136,7 +1136,7 @@ func TestOOMMaliciousInput(t *testing.T) {
},
}
for i, test := range oomTests {
def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
def := fmt.Sprintf(`[{ "name" : "method", "type": "function", "outputs": %s}]`, test.def)
abi, err := JSON(strings.NewReader(def))
if err != nil {
t.Fatalf("invalid ABI definition %s: %v", def, err)

7
accounts/manager.go
View file

@ -141,6 +141,11 @@ func (am *Manager) Wallets() []Wallet {
am.lock.RLock()
defer am.lock.RUnlock()
return am.walletsNoLock()
}
// walletsNoLock returns all registered wallets. Callers must hold am.lock.
func (am *Manager) walletsNoLock() []Wallet {
cpy := make([]Wallet, len(am.wallets))
copy(cpy, am.wallets)
return cpy
@ -155,7 +160,7 @@ func (am *Manager) Wallet(url string) (Wallet, error) {
if err != nil {
return nil, err
}
for _, wallet := range am.Wallets() {
for _, wallet := range am.walletsNoLock() {
if wallet.URL() == parsed {
return wallet, nil
}

6
accounts/scwallet/wallet.go
View file

@ -312,15 +312,15 @@ func (w *Wallet) Status() (string, error) {
}
switch {
case !w.session.verified && status.PinRetryCount == 0 && status.PukRetryCount == 0:
return fmt.Sprintf("Bricked, waiting for full wipe"), nil
return "Bricked, waiting for full wipe", nil
case !w.session.verified && status.PinRetryCount == 0:
return fmt.Sprintf("Blocked, waiting for PUK (%d attempts left) and new PIN", status.PukRetryCount), nil
case !w.session.verified:
return fmt.Sprintf("Locked, waiting for PIN (%d attempts left)", status.PinRetryCount), nil
case !status.Initialized:
return fmt.Sprintf("Empty, waiting for initialization"), nil
return "Empty, waiting for initialization", nil
default:
return fmt.Sprintf("Online"), nil
return "Online", nil
}
}

5
appveyor.yml
View file

@ -6,6 +6,7 @@ clone_depth: 5
version: "{branch}.{build}"
environment:
global:
GO111MODULE: on
GOPATH: C:\gopath
CC: gcc.exe
matrix:
@ -23,8 +24,8 @@ environment:
install:
- git submodule update --init
- rmdir C:\go /s /q
- appveyor DownloadFile https://dl.google.com/go/go1.13.4.windows-%GETH_ARCH%.zip
- 7z x go1.13.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- appveyor DownloadFile https://dl.google.com/go/go1.14.2.windows-%GETH_ARCH%.zip
- 7z x go1.14.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version

34
build/checksums.txt
View file

@ -1,19 +1,21 @@
# This file contains sha256 checksums of optional build dependencies.
aae5be954bdc40bcf8006eb77e8d8a5dde412722bc8effcdaf9772620d06420c go1.13.6.src.tar.gz
98de84e69726a66da7b4e58eac41b99cbe274d7e8906eeb8a5b7eb0aadee7f7c go1.14.2.src.tar.gz
478994633b0f5121a7a8d4f368078093e21014fdc7fb2c0ceeae63668c13c5b6 golangci-lint-1.22.2-freebsd-amd64.tar.gz
fcf80824c21567eb0871055711bf9bdca91cf9a081122e2a45f1d11fed754600 golangci-lint-1.22.2-darwin-amd64.tar.gz
cda85c72fc128b2ea0ae05baea7b91172c63aea34064829f65285f1dd536f1e0 golangci-lint-1.22.2-windows-386.zip
94f04899f620aadc9c1524e5482e415efdbd993fa2b2918c4fec2798f030ac1c golangci-lint-1.22.2-linux-armv7.tar.gz
0e72a87d71edde00b6e37e84a99841833ad55fee83e20d21130a7a622b2860bb golangci-lint-1.22.2-freebsd-386.tar.gz
86def2f31fe8fd7c05674104ed2a4bef3e44b7132b93c6ad2f52f198b3d01801 golangci-lint-1.22.2-linux-s390x.tar.gz
b0df4546d36be94e8107733ba290b98dd9b7e41a42d3fb202e87fc7e4ee800c3 golangci-lint-1.22.2-freebsd-armv6.tar.gz
3d45958dcf6a8d195086d2fced1a21db42a90815dfd156d180efa62dbdda6724 golangci-lint-1.22.2-darwin-386.tar.gz
7ee29f35c74fab017a454237990c74d984ce3855960f2c10509238992bb781f9 golangci-lint-1.22.2-linux-arm64.tar.gz
52086ac52a502b68578e58e35d3964f127c16d7a90b9ffcb399a004d055ded51 golangci-lint-1.22.2-linux-386.tar.gz
c2e4df1fab2ae53762f9baac6041503eeeaa968ce38ea41779f7cb526751c667 golangci-lint-1.22.2-windows-amd64.zip
109d38cdc89f271392f5a138d6782657157f9f496fd4801956efa2d0428e0cbe golangci-lint-1.22.2-linux-amd64.tar.gz
f08aae4868d4828c8f07deb0dcd941a1da695b97e58d15e9f3d1d07dcc7a0c84 golangci-lint-1.22.2-linux-armv6.tar.gz
37af03d9c144d527cb15c46a07e6a22d3f62b5491e34ad6f3bfe6bb0b0b597d4 golangci-lint-1.22.2-linux-ppc64le.tar.gz
251a1081d53944f1d5f86216d752837b23079f90605c9d1cc628da1ffcd2e749 golangci-lint-1.22.2-freebsd-armv7.tar.gz
aeaa5498682246b87d0b77ece283897348ea03d98e816760a074058bfca60b2a golangci-lint-1.24.0-windows-amd64.zip
7e854a70d449fe77b7a91583ec88c8603eb3bf96c45d52797dc4ba3f2f278dbe golangci-lint-1.24.0-darwin-386.tar.gz
835101fae192c3a2e7a51cb19d5ac3e1a40b0e311955e89bc21d61de78635979 golangci-lint-1.24.0-linux-armv6.tar.gz
a041a6e6a61c9ff3dbe58673af13ea00c76bcd462abede0ade645808e97cdd6d golangci-lint-1.24.0-windows-386.zip
7cc73eb9ca02b7a766c72b913f8080401862b10e7bb90c09b085415a81f21609 golangci-lint-1.24.0-freebsd-armv6.tar.gz
537bb2186987b5e68ad4e8829230557f26087c3028eb736dea1662a851bad73d golangci-lint-1.24.0-linux-armv7.tar.gz
8cb1bc1e63d8f0d9b71fcb10b38887e1646a6b8a120ded2e0cd7c3284528f633 golangci-lint-1.24.0-linux-mips64.tar.gz
095d3f8bf7fc431739861574d0b58d411a617df2ed5698ce5ae5ecc66d23d44d golangci-lint-1.24.0-freebsd-armv7.tar.gz
e245df27cec3827aef9e7afbac59e92816978ee3b64f84f7b88562ff4b2ac225 golangci-lint-1.24.0-linux-arm64.tar.gz
35d6d5927e19f0577cf527f0e4441dbb37701d87e8cf729c98a510fce397fbf7 golangci-lint-1.24.0-linux-ppc64le.tar.gz
a1ed66353b8ceb575d78db3051491bce3ac1560e469a9bc87e8554486fec7dfe golangci-lint-1.24.0-freebsd-386.tar.gz
241ca454102e909de04957ff8a5754c757cefa255758b3e1fba8a4533d19d179 golangci-lint-1.24.0-linux-amd64.tar.gz
ff488423db01a0ec8ffbe4e1d65ef1be6a8d5e6d7930cf380ce8aaf714125470 golangci-lint-1.24.0-linux-386.tar.gz
f05af56f15ebbcf77663a8955d1e39009b584ce8ea4c5583669369d80353a113 golangci-lint-1.24.0-darwin-amd64.tar.gz
b0096796c0ffcd6c350a2ec006100e7ef5f0597b43a204349d4f997273fb32a7 golangci-lint-1.24.0-freebsd-amd64.tar.gz
c9c2867380e85628813f1f7d1c3cfc6c6f7931e89bea86f567ff451b8cdb6654 golangci-lint-1.24.0-linux-mips64le.tar.gz
2feb97fa61c934aa3eba9bc104ab5dd8fb946791d58e64060e8857e800eeae0b golangci-lint-1.24.0-linux-s390x.tar.gz

2
build/ci.go
View file

@ -356,7 +356,7 @@ func doLint(cmdline []string) {
// downloadLinter downloads and unpacks golangci-lint.
func downloadLinter(cachedir string) string {
const version = "1.22.2"
const version = "1.24.0"
csdb := build.MustLoadChecksums("build/checksums.txt")
base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)

74
cmd/abidump/main.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"encoding/hex"
"flag"
"fmt"
"os"
"strings"
"github.com/ethereum/go-ethereum/signer/core"
"github.com/ethereum/go-ethereum/signer/fourbyte"
)
func init() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "Usage:", os.Args[0], "<hexdata>")
flag.PrintDefaults()
fmt.Fprintln(os.Stderr, `
Parses the given ABI data and tries to interpret it from the fourbyte database.`)
}
}
func parse(data []byte) {
db, err := fourbyte.New()
if err != nil {
die(err)
}
messages := core.ValidationMessages{}
db.ValidateCallData(nil, data, &messages)
for _, m := range messages.Messages {
fmt.Printf("%v: %v\n", m.Typ, m.Message)
}
}
// Example
// ./abidump a9059cbb000000000000000000000000ea0e2dc7d65a50e77fc7e84bff3fd2a9e781ff5c0000000000000000000000000000000000000000000000015af1d78b58c40000
func main() {
flag.Parse()
switch {
case flag.NArg() == 1:
hexdata := flag.Arg(0)
data, err := hex.DecodeString(strings.TrimPrefix(hexdata, "0x"))
if err != nil {
die(err)
}
parse(data)
default:
fmt.Fprintln(os.Stderr, "Error: one argument needed")
flag.Usage()
os.Exit(2)
}
}
func die(args ...interface{}) {
fmt.Fprintln(os.Stderr, args...)
os.Exit(1)
}

103
cmd/checkpoint-admin/README.md Normal file
View file

@ -0,0 +1,103 @@
## Checkpoint-admin
Checkpoint-admin is a tool for updating checkpoint oracle status. It provides a series of functions including deploying checkpoint oracle contract, signing for new checkpoints, and updating checkpoints in the checkpoint oracle contract.
### Checkpoint
In the LES protocol, there is an important concept called checkpoint. In simple terms, whenever a certain number of blocks are generated on the blockchain, a new checkpoint is generated which contains some important information such as
* Block hash at checkpoint
* Canonical hash trie root at checkpoint
* Bloom trie root at checkpoint
*For a more detailed introduction to checkpoint, please see the LES [spec](https://github.com/ethereum/devp2p/blob/master/caps/les.md).*
Using this information, light clients can skip all historical block headers when synchronizing data and start synchronization from this checkpoint. Therefore, as long as the light client can obtain some latest and correct checkpoints, the amount of data and time for synchronization will be greatly reduced.
However, from a security perspective, the most critical step in a synchronization algorithm based on checkpoints is to determine whether the checkpoint used by the light client is correct. Otherwise, all blockchain data synchronized based on this checkpoint may be wrong. For this we provide two different ways to ensure the correctness of the checkpoint used by the light client.
#### Hardcoded checkpoint
There are several hardcoded checkpoints in the [source code](https://github.com/ethereum/go-ethereum/blob/master/params/config.go#L38) of the go-ethereum project. These checkpoints are updated by go-ethereum developers when new versions of software are released. Because light client users trust Geth developers to some extent, hardcoded checkpoints in the code can also be considered correct.
#### Checkpoint oracle
Hardcoded checkpoints can solve the problem of verifying the correctness of checkpoints (although this is a more centralized solution). But the pain point of this solution is that developers can only update checkpoints when a new version of software is released. In addition, light client users usually do not keep the Geth version they use always up to date. So hardcoded checkpoints used by users are generally stale. Therefore, it still needs to download a large amount of blockchain data during synchronization.
Checkpoint oracle is a more flexible solution. In simple terms, this is a smart contract that is deployed on the blockchain. The smart contract records several designated trusted signers. Whenever enough trusted signers have issued their signatures for the same checkpoint, it can be considered that the checkpoint has been authenticated by the signers. Checkpoints authenticated by trusted signers can be considered correct.
So this way, even without updating the software version, as long as the trusted signers regularly update the checkpoint in oracle on time, the light client can always use the latest and verified checkpoint for data synchronization.
### Usage
Checkpoint-admin is a command line tool designed for checkpoint oracle. Users can easily deploy contracts and update checkpoints through this tool.
#### Install
```shell
go get github.com/ethereum/go-ethereum/cmd/checkpoint-admin
```
#### Deploy
Deploy checkpoint oracle contract. `--signers` indicates the specified trusted signer, and `--threshold` indicates the minimum number of signatures required by trusted signers to update a checkpoint.
```shell
checkpoint-admin deploy --rpc <NODE_RPC_ENDPOINT> --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_TX> --signers <TRUSTED_SIGNER_LIST> --threshold 1
```
It is worth noting that checkpoint-admin only supports clef as a signer for transactions and plain text(checkpoint). For more clef usage, please see the clef [tutorial](https://geth.ethereum.org/docs/clef/tutorial) .
#### Sign
Checkpoint-admin provides two different modes of signing. You can automatically obtain the current stable checkpoint and sign it interactively, and you can also use the information provided by the command line flags to sign checkpoint offline.
**Interactive mode**
```shell
checkpoint-admin sign --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_CHECKPOINT> --rpc <NODE_RPC_ENDPOINT>
```
*It is worth noting that the connected Geth node can be a fullnode or a light client. If it is fullnode, you must enable the LES protocol. E.G. add `--light.serv 50` to the startup command line flags*.
**Offline mode**
```shell
checkpoint-admin sign --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_CHECKPOINT> --index <CHECKPOINT_INDEX> --hash <CHECKPOINT_HASH> --oracle <CHECKPOINT_ORACLE_ADDRESS>
```
*CHECKPOINT_HASH is obtained based on this [calculation method](https://github.com/ethereum/go-ethereum/blob/master/params/config.go#L251).*
#### Publish
Collect enough signatures from different trusted signers for the same checkpoint and submit them to oracle to update the "authenticated" checkpoint in the contract.
```shell
checkpoint-admin publish --clef <CLEF_ENDPOINT> --rpc <NODE_RPC_ENDPOINT> --signer <SIGNER_TO_SIGN_TX> --index <CHECKPOINT_INDEX> --signatures <CHECKPOINT_SIGNATURE_LIST>
```
#### Status query
Check the latest status of checkpoint oracle.
```shell
checkpoint-admin status --rpc <NODE_RPC_ENDPOINT>
```
### Enable checkpoint oracle in your private network
Currently, only the Ethereum mainnet and the default supported test networks (ropsten, rinkeby, goerli) activate this feature. If you want to activate this feature in your private network, you can overwrite the relevant checkpoint oracle settings through the configuration file after deploying the oracle contract.
* Get your node configuration file `geth dumpconfig OTHER_COMMAND_LINE_OPTIONS > config.toml`
* Edit the configuration file and add the following information
```toml
[Eth.CheckpointOracle]
Address = CHECKPOINT_ORACLE_ADDRESS
Signers = [TRUSTED_SIGNER_1, ..., TRUSTED_SIGNER_N]
Threshold = THRESHOLD
```
* Start geth with the modified configuration file
*In the private network, all fullnodes and light clients need to be started using the same checkpoint oracle settings.*

14
cmd/clef/docs/setup.md
View file

@ -34,7 +34,7 @@ There are two ways that this can be achieved: integrated via Qubes or integrated
#### 1. Qubes Integrated
Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request
Qubes provides a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request
to another qube. The OS then asks the user if the call is permitted.
![Example](qubes/qrexec-example.png)
@ -48,7 +48,7 @@ This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented.
![Clef via qrexec](qubes/clef_qubes_qrexec.png)
On the `target` qubes, we need to define the rpc service.
On the `target` qubes, we need to define the RPC service.
[qubes.Clefsign](qubes/qubes.Clefsign):
@ -135,11 +135,11 @@ $ cat newaccnt.json
$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign
```
This should pop up first a dialog to allow the IPC call:
A dialog should pop up first to allow the IPC call:
![one](qubes/qubes_newaccount-1.png)
Followed by a GTK-dialog to approve the operation
Followed by a GTK-dialog to approve the operation:
![two](qubes/qubes_newaccount-2.png)
@ -169,7 +169,7 @@ However, it comes with a couple of drawbacks:
- The `Origin` header must be forwarded
- Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header,
since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address.
- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups:
- Even with a policy in place to allow RPC calls between `caller` and `target`, there will be several popups:
- One qubes-specific where the user specifies the `target` vm
- One clef-specific to approve the transaction
@ -177,7 +177,7 @@ However, it comes with a couple of drawbacks:
#### 2. Network integrated
The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible
form other qubes.
from other qubes.
![Clef via http](qubes/clef_qubes_http.png)
@ -193,6 +193,6 @@ to your computer. Over this new network interface, you can SSH into the device.
Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only
ever connects to a local network between your computer and the device itself.
Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access
Needless to say, while this model should be fairly secure against remote attacks, an attacker with physical access
to the USB Armory would trivially be able to extract the contents of the device filesystem.

11
cmd/clef/intapi_changelog.md
View file

@ -10,6 +10,17 @@ TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
### 7.0.1
Added `clef_New` to the internal API calleable from a UI.
> `New` creates a new password protected Account. The private key is protected with
> the given password. Users are responsible to backup the private key that is stored
> in the keystore location that was specified when this API was created.
> This method is the same as New on the external API, the difference being that
> this implementation does not ask for confirmation, since it's initiated by
> the user
### 7.0.0
- The `message` field was renamed to `messages` in all data signing request methods to better reflect that it's a list, not a value.

61
cmd/clef/main.go
View file

@ -187,6 +187,21 @@ The setpw command stores a password for a given address (keyfile).
Description: `
The delpw command removes a password for a given address (keyfile).
`}
newAccountCommand = cli.Command{
Action: utils.MigrateFlags(newAccount),
Name: "newaccount",
Usage: "Create a new account",
ArgsUsage: "",
Flags: []cli.Flag{
logLevelFlag,
keystoreFlag,
utils.LightKDFFlag,
},
Description: `
The newaccount command creates a new keystore-backed account. It is a convenience-method
which can be used in lieu of an external UI.`,
}
gendocCommand = cli.Command{
Action: GenDoc,
Name: "gendoc",
@ -222,7 +237,12 @@ func init() {
advancedMode,
}
app.Action = signer
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, gendocCommand}
app.Commands = []cli.Command{initCommand,
attestCommand,
setCredentialCommand,
delCredentialCommand,
newAccountCommand,
gendocCommand}
cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate
}
@ -382,6 +402,31 @@ func removeCredential(ctx *cli.Context) error {
return nil
}
func newAccount(c *cli.Context) error {
if err := initialize(c); err != nil {
return err
}
// The newaccount is meant for users using the CLI, since 'real' external
// UIs can use the UI-api instead. So we'll just use the native CLI UI here.
var (
ui = core.NewCommandlineUI()
pwStorage storage.Storage = &storage.NoStorage{}
ksLoc = c.GlobalString(keystoreFlag.Name)
lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
)
log.Info("Starting clef", "keystore", ksLoc, "light-kdf", lightKdf)
am := core.StartClefAccountManager(ksLoc, true, lightKdf, "")
// This gives is us access to the external API
apiImpl := core.NewSignerAPI(am, 0, true, ui, nil, false, pwStorage)
// This gives us access to the internal API
internalApi := core.NewUIServerAPI(apiImpl)
addr, err := internalApi.New(context.Background())
if err == nil {
fmt.Printf("Generated account %v\n", addr.String())
}
return err
}
func initialize(c *cli.Context) error {
// Set up the logger to print everything
logOutput := os.Stdout
@ -457,7 +502,6 @@ func signer(c *cli.Context) error {
api core.ExternalAPI
pwStorage storage.Storage = &storage.NoStorage{}
)
configDir := c.GlobalString(configdirFlag.Name)
if stretchedKey, err := readMasterKey(c, ui); err != nil {
log.Warn("Failed to open master, rules disabled", "err", err)
@ -539,18 +583,25 @@ func signer(c *cli.Context) error {
vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
srv := rpc.NewServer()
err := node.RegisterApisFromWhitelist(rpcAPI, []string{"account"}, srv, false)
if err != nil {
utils.Fatalf("Could not register API: %w", err)
}
handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
// start http server
httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
listener, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler)
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}
extapiURL = fmt.Sprintf("http://%s", httpEndpoint)
extapiURL = fmt.Sprintf("http://%v/", listener.Addr())
log.Info("HTTP endpoint opened", "url", extapiURL)
defer func() {
listener.Close()
log.Info("HTTP endpoint closed", "url", httpEndpoint)
log.Info("HTTP endpoint closed", "url", extapiURL)
}()
}
if !c.GlobalBool(utils.IPCDisabledFlag.Name) {

2
cmd/clef/tutorial.md
View file

@ -1,6 +1,6 @@
## Initializing Clef
First thing's first, Clef needs to store some data itself. Since that data might be sensitive (passwords, signing rules, accounts), Clef's entire storage is encrypted. To support encrypting data, the first step is to initialize Clef with a random master seed, itself too encrypted with your chosen password:
First things first, Clef needs to store some data itself. Since that data might be sensitive (passwords, signing rules, accounts), Clef's entire storage is encrypted. To support encrypting data, the first step is to initialize Clef with a random master seed, itself too encrypted with your chosen password:
```text
$ clef init

10
cmd/devp2p/crawl.go
View file

@ -20,14 +20,13 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/enode"
)
type crawler struct {
input nodeSet
output nodeSet
disc *discover.UDPv4
disc resolver
iters []enode.Iterator
inputIter enode.Iterator
ch chan *enode.Node
@ -37,7 +36,11 @@ type crawler struct {
revalidateInterval time.Duration
}
func newCrawler(input nodeSet, disc *discover.UDPv4, iters ...enode.Iterator) *crawler {
type resolver interface {
RequestENR(*enode.Node) (*enode.Node, error)
}
func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler {
c := &crawler{
input: input,
output: make(nodeSet, len(input)),
@ -63,6 +66,7 @@ func (c *crawler) run(timeout time.Duration) nodeSet {
doneCh = make(chan enode.Iterator, len(c.iters))
liveIters = len(c.iters)
)
defer timeoutTimer.Stop()
for _, it := range c.iters {
go c.runIterator(doneCh, it)
}

105
cmd/devp2p/discv4cmd.go
View file

@ -81,6 +81,18 @@ var (
Name: "bootnodes",
Usage: "Comma separated nodes used for bootstrapping",
}
nodekeyFlag = cli.StringFlag{
Name: "nodekey",
Usage: "Hex-encoded node key",
}
nodedbFlag = cli.StringFlag{
Name: "nodedb",
Usage: "Nodes database location",
}
listenAddrFlag = cli.StringFlag{
Name: "addr",
Usage: "Listening address",
}
crawlTimeoutFlag = cli.DurationFlag{
Name: "timeout",
Usage: "Time limit for the crawl.",
@ -172,6 +184,62 @@ func discv4Crawl(ctx *cli.Context) error {
return nil
}
// startV4 starts an ephemeral discovery V4 node.
func startV4(ctx *cli.Context) *discover.UDPv4 {
ln, config := makeDiscoveryConfig(ctx)
socket := listen(ln, ctx.String(listenAddrFlag.Name))
disc, err := discover.ListenV4(socket, ln, config)
if err != nil {
exit(err)
}
return disc
}
func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) {
var cfg discover.Config
if ctx.IsSet(nodekeyFlag.Name) {
key, err := crypto.HexToECDSA(ctx.String(nodekeyFlag.Name))
if err != nil {
exit(fmt.Errorf("-%s: %v", nodekeyFlag.Name, err))
}
cfg.PrivateKey = key
} else {
cfg.PrivateKey, _ = crypto.GenerateKey()
}
if commandHasFlag(ctx, bootnodesFlag) {
bn, err := parseBootnodes(ctx)
if err != nil {
exit(err)
}
cfg.Bootnodes = bn
}
dbpath := ctx.String(nodedbFlag.Name)
db, err := enode.OpenDB(dbpath)
if err != nil {
exit(err)
}
ln := enode.NewLocalNode(db, cfg.PrivateKey)
return ln, cfg
}
func listen(ln *enode.LocalNode, addr string) *net.UDPConn {
if addr == "" {
addr = "0.0.0.0:0"
}
socket, err := net.ListenPacket("udp4", addr)
if err != nil {
exit(err)
}
usocket := socket.(*net.UDPConn)
uaddr := socket.LocalAddr().(*net.UDPAddr)
ln.SetFallbackIP(net.IP{127, 0, 0, 1})
ln.SetFallbackUDP(uaddr.Port)
return usocket
}
func parseBootnodes(ctx *cli.Context) ([]*enode.Node, error) {
s := params.RinkebyBootnodes
if ctx.IsSet(bootnodesFlag.Name) {
@ -187,40 +255,3 @@ func parseBootnodes(ctx *cli.Context) ([]*enode.Node, error) {
}
return nodes, nil
}
// startV4 starts an ephemeral discovery V4 node.
func startV4(ctx *cli.Context) *discover.UDPv4 {
socket, ln, cfg, err := listen()
if err != nil {
exit(err)
}
if commandHasFlag(ctx, bootnodesFlag) {
bn, err := parseBootnodes(ctx)
if err != nil {
exit(err)
}
cfg.Bootnodes = bn
}
disc, err := discover.ListenV4(socket, ln, cfg)
if err != nil {
exit(err)
}
return disc
}
func listen() (*net.UDPConn, *enode.LocalNode, discover.Config, error) {
var cfg discover.Config
cfg.PrivateKey, _ = crypto.GenerateKey()
db, _ := enode.OpenDB("")
ln := enode.NewLocalNode(db, cfg.PrivateKey)
socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{0, 0, 0, 0}})
if err != nil {
db.Close()
return nil, nil, cfg, err
}
addr := socket.LocalAddr().(*net.UDPAddr)
ln.SetFallbackIP(net.IP{127, 0, 0, 1})
ln.SetFallbackUDP(addr.Port)
return socket, ln, cfg, nil
}

123
cmd/devp2p/discv5cmd.go Normal file
View file

@ -0,0 +1,123 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p/discover"
"gopkg.in/urfave/cli.v1"
)
var (
discv5Command = cli.Command{
Name: "discv5",
Usage: "Node Discovery v5 tools",
Subcommands: []cli.Command{
discv5PingCommand,
discv5ResolveCommand,
discv5CrawlCommand,
discv5ListenCommand,
},
}
discv5PingCommand = cli.Command{
Name: "ping",
Usage: "Sends ping to a node",
Action: discv5Ping,
}
discv5ResolveCommand = cli.Command{
Name: "resolve",
Usage: "Finds a node in the DHT",
Action: discv5Resolve,
Flags: []cli.Flag{bootnodesFlag},
}
discv5CrawlCommand = cli.Command{
Name: "crawl",
Usage: "Updates a nodes.json file with random nodes found in the DHT",
Action: discv5Crawl,
Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag},
}
discv5ListenCommand = cli.Command{
Name: "listen",
Usage: "Runs a node",
Action: discv5Listen,
Flags: []cli.Flag{
bootnodesFlag,
nodekeyFlag,
nodedbFlag,
listenAddrFlag,
},
}
)
func discv5Ping(ctx *cli.Context) error {
n := getNodeArg(ctx)
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Ping(n))
return nil
}
func discv5Resolve(ctx *cli.Context) error {
n := getNodeArg(ctx)
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Resolve(n))
return nil
}
func discv5Crawl(ctx *cli.Context) error {
if ctx.NArg() < 1 {
return fmt.Errorf("need nodes file as argument")
}
nodesFile := ctx.Args().First()
var inputSet nodeSet
if common.FileExist(nodesFile) {
inputSet = loadNodesJSON(nodesFile)
}
disc := startV5(ctx)
defer disc.Close()
c := newCrawler(inputSet, disc, disc.RandomNodes())
c.revalidateInterval = 10 * time.Minute
output := c.run(ctx.Duration(crawlTimeoutFlag.Name))
writeNodesJSON(nodesFile, output)
return nil
}
func discv5Listen(ctx *cli.Context) error {
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Self())
select {}
}
// startV5 starts an ephemeral discovery v5 node.
func startV5(ctx *cli.Context) *discover.UDPv5 {
ln, config := makeDiscoveryConfig(ctx)
socket := listen(ln, ctx.String(listenAddrFlag.Name))
disc, err := discover.ListenV5(socket, ln, config)
if err != nil {
exit(err)
}
return disc
}

4
cmd/devp2p/dns_cloudflare.go
View file

@ -130,9 +130,9 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
if !exists {
// Entry is unknown, push a new one to Cloudflare.
log.Info(fmt.Sprintf("Creating %s = %q", path, val))
ttl := 1
ttl := rootTTL
if path != name {
ttl = 2147483647 // Max TTL permitted by Cloudflare
ttl = treeNodeTTL // Max TTL permitted by Cloudflare
}
_, err = c.CreateDNSRecord(c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl})
} else if old.Content != val {

170
cmd/devp2p/dns_route53.go
View file

@ -19,6 +19,7 @@ package main
import (
"errors"
"fmt"
"sort"
"strconv"
"strings"
@ -31,6 +32,14 @@ import (
"gopkg.in/urfave/cli.v1"
)
const (
// Route53 limits change sets to 32k of 'RDATA size'. Change sets are also limited to
// 1000 items. UPSERTs count double.
// https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets
route53ChangeSizeLimit = 32000
route53ChangeCountLimit = 1000
)
var (
route53AccessKeyFlag = cli.StringFlag{
Name: "access-key-id",
@ -53,6 +62,11 @@ type route53Client struct {
zoneID string
}
type recordSet struct {
values []string
ttl int64
}
// newRoute53Client sets up a Route53 API client from command line flags.
func newRoute53Client(ctx *cli.Context) *route53Client {
akey := ctx.String(route53AccessKeyFlag.Name)
@ -78,31 +92,39 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error {
}
// Compute DNS changes.
records := t.ToTXT(name)
changes, err := c.computeChanges(name, records)
existing, err := c.collectRecords(name)
if err != nil {
return err
}
log.Info(fmt.Sprintf("Found %d TXT records", len(existing)))
records := t.ToTXT(name)
changes := c.computeChanges(name, records, existing)
if len(changes) == 0 {
log.Info("No DNS changes needed")
return nil
}
// Submit change request.
log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes)))
batch := new(route53.ChangeBatch)
batch.SetChanges(changes)
batch.SetComment(fmt.Sprintf("enrtree update of %s at seq %d", name, t.Seq()))
req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch}
resp, err := c.api.ChangeResourceRecordSets(req)
if err != nil {
return err
}
// Submit change batches.
batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit)
for i, changes := range batches {
log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes)))
batch := new(route53.ChangeBatch)
batch.SetChanges(changes)
batch.SetComment(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq()))
req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch}
resp, err := c.api.ChangeResourceRecordSets(req)
if err != nil {
return err
}
// Wait for the change to be applied.
log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id))
wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id}
return c.api.WaitUntilResourceRecordSetsChanged(wreq)
log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id))
wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id}
if err := c.api.WaitUntilResourceRecordSetsChanged(wreq); err != nil {
return err
}
}
return nil
}
// checkZone verifies zone information for the given domain.
@ -137,7 +159,7 @@ func (c *route53Client) findZoneID(name string) (string, error) {
}
// computeChanges creates DNS changes for the given record.
func (c *route53Client) computeChanges(name string, records map[string]string) ([]*route53.Change, error) {
func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []*route53.Change {
// Convert all names to lowercase.
lrecords := make(map[string]string, len(records))
for name, r := range records {
@ -145,27 +167,20 @@ func (c *route53Client) computeChanges(name string, records map[string]string) (
}
records = lrecords
// Get existing records.
existing, err := c.collectRecords(name)
if err != nil {
return nil, err
}
log.Info(fmt.Sprintf("Found %d TXT records", len(existing)))
var changes []*route53.Change
for path, val := range records {
ttl := 1
ttl := int64(rootTTL)
if path != name {
ttl = 2147483647
ttl = int64(treeNodeTTL)
}
prevRecords, exists := existing[path]
prevValue := combineTXT(prevRecords)
prevValue := strings.Join(prevRecords.values, "")
if !exists {
// Entry is unknown, push a new one
log.Info(fmt.Sprintf("Creating %s = %q", path, val))
changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val)))
} else if prevValue != val {
} else if prevValue != val || prevRecords.ttl != ttl {
// Entry already exists, only change its content.
log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val))
changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val)))
@ -175,32 +190,91 @@ func (c *route53Client) computeChanges(name string, records map[string]string) (
}
// Iterate over the old records and delete anything stale.
for path, values := range existing {
for path, set := range existing {
if _, ok := records[path]; ok {
continue
}
// Stale entry, nuke it.
log.Info(fmt.Sprintf("Deleting %s = %q", path, combineTXT(values)))
changes = append(changes, newTXTChange("DELETE", path, 1, values))
log.Info(fmt.Sprintf("Deleting %s = %q", path, strings.Join(set.values, "")))
changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...))
}
return changes, nil
sortChanges(changes)
return changes
}
// sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order.
func sortChanges(changes []*route53.Change) {
score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3}
sort.Slice(changes, func(i, j int) bool {
if *changes[i].Action == *changes[j].Action {
return *changes[i].ResourceRecordSet.Name < *changes[j].ResourceRecordSet.Name
}
return score[*changes[i].Action] < score[*changes[j].Action]
})
}
// splitChanges splits up DNS changes such that each change batch
// is smaller than the given RDATA limit.
func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*route53.Change {
var (
batches [][]*route53.Change
batchSize int
batchCount int
)
for _, ch := range changes {
// Start new batch if this change pushes the current one over the limit.
count := changeCount(ch)
size := changeSize(ch) * count
overSize := batchSize+size > sizeLimit
overCount := batchCount+count > countLimit
if len(batches) == 0 || overSize || overCount {
batches = append(batches, nil)
batchSize = 0
batchCount = 0
}
batches[len(batches)-1] = append(batches[len(batches)-1], ch)
batchSize += size
batchCount += count
}
return batches
}
// changeSize returns the RDATA size of a DNS change.
func changeSize(ch *route53.Change) int {
size := 0
for _, rr := range ch.ResourceRecordSet.ResourceRecords {
if rr.Value != nil {
size += len(*rr.Value)
}
}
return size
}
func changeCount(ch *route53.Change) int {
if *ch.Action == "UPSERT" {
return 2
}
return 1
}
// collectRecords collects all TXT records below the given name.
func (c *route53Client) collectRecords(name string) (map[string][]string, error) {
func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) {
log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID))
var req route53.ListResourceRecordSetsInput
req.SetHostedZoneId(c.zoneID)
existing := make(map[string][]string)
existing := make(map[string]recordSet)
err := c.api.ListResourceRecordSetsPages(&req, func(resp *route53.ListResourceRecordSetsOutput, last bool) bool {
for _, set := range resp.ResourceRecordSets {
if !isSubdomain(*set.Name, name) || *set.Type != "TXT" {
continue
}
name := strings.TrimSuffix(*set.Name, ".")
s := recordSet{ttl: *set.TTL}
for _, rec := range set.ResourceRecords {
existing[name] = append(existing[name], *rec.Value)
s.values = append(s.values, *rec.Value)
}
name := strings.TrimSuffix(*set.Name, ".")
existing[name] = s
}
return true
})
@ -208,7 +282,7 @@ func (c *route53Client) collectRecords(name string) (map[string][]string, error)
}
// newTXTChange creates a change to a TXT record.
func newTXTChange(action, name string, ttl int, values []string) *route53.Change {
func newTXTChange(action, name string, ttl int64, values ...string) *route53.Change {
var c route53.Change
var r route53.ResourceRecordSet
var rrs []*route53.ResourceRecord
@ -219,7 +293,7 @@ func newTXTChange(action, name string, ttl int, values []string) *route53.Change
}
r.SetType("TXT")
r.SetName(name)
r.SetTTL(int64(ttl))
r.SetTTL(ttl)
r.SetResourceRecords(rrs)
c.SetAction(action)
c.SetResourceRecordSet(&r)
@ -233,28 +307,16 @@ func isSubdomain(name, domain string) bool {
return strings.HasSuffix("."+name, "."+domain)
}
// combineTXT concatenates the given quoted strings into a single unquoted string.
func combineTXT(values []string) string {
result := ""
for _, v := range values {
if v[0] == '"' {
v = v[1 : len(v)-1]
}
result += v
}
return result
}
// splitTXT splits value into a list of quoted 255-character strings.
func splitTXT(value string) []string {
var result []string
func splitTXT(value string) string {
var result strings.Builder
for len(value) > 0 {
rlen := len(value)
if rlen > 253 {
rlen = 253
}
result = append(result, strconv.Quote(value[:rlen]))
result.WriteString(strconv.Quote(value[:rlen]))
value = value[rlen:]
}
return result
return result.String()
}

166
cmd/devp2p/dns_route53_test.go Normal file
View file

@ -0,0 +1,166 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"reflect"
"testing"
"github.com/aws/aws-sdk-go/service/route53"
)
// This test checks that computeChanges/splitChanges create DNS changes in
// leaf-added -> root-changed -> leaf-deleted order.
func TestRoute53ChangeSort(t *testing.T) {
testTree0 := map[string]recordSet{
"2kfjogvxdqtxxugbh7gs7naaai.n": {ttl: 3333, values: []string{
`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`,
}},
"fdxn3sn67na5dka4j2gok7bvqi.n": {ttl: treeNodeTTL, values: []string{`"enrtree-branch:"`}},
"n": {ttl: rootTTL, values: []string{`"enrtree-root:v1 e=2KFJOGVXDQTXXUGBH7GS7NAAAI l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=0 sig=v_-J_q_9ICQg5ztExFvLQhDBGMb0lZPJLhe3ts9LAcgqhOhtT3YFJsl8BWNDSwGtamUdR-9xl88_w-X42SVpjwE"`}},
}
testTree1 := map[string]string{
"n": "enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA",
"C7HRFPF3BLGF3YR4DY5KX3SMBE.n": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org",
"JWXYDBPXYWG6FX3GMDIBFA6CJ4.n": "enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24",
"2XS2367YHAXJFGLZHVAWLQD4ZY.n": "enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA",
"H4FHT4B454P6UXFD7JCYQ5PWDY.n": "enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI",
"MHTDO6TMUBRIA2XWG5LUDACK24.n": "enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o",
}
wantChanges := []*route53.Change{
{
Action: sp("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("2xs2367yhaxjfglzhvawlqd4zy.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
{
Action: sp("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("c7hrfpf3blgf3yr4dy5kx3smbe.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
{
Action: sp("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("h4fht4b454p6uxfd7jcyq5pwdy.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
{
Action: sp("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("jwxydbpxywg6fx3gmdibfa6cj4.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
{
Action: sp("CREATE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("mhtdo6tmubria2xwg5ludack24.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
{
Action: sp("UPSERT"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`),
}},
TTL: ip(rootTTL),
Type: sp("TXT"),
},
},
{
Action: sp("DELETE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("2kfjogvxdqtxxugbh7gs7naaai.n"),
ResourceRecords: []*route53.ResourceRecord{
{Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)},
},
TTL: ip(3333),
Type: sp("TXT"),
},
},
{
Action: sp("DELETE"),
ResourceRecordSet: &route53.ResourceRecordSet{
Name: sp("fdxn3sn67na5dka4j2gok7bvqi.n"),
ResourceRecords: []*route53.ResourceRecord{{
Value: sp(`"enrtree-branch:"`),
}},
TTL: ip(treeNodeTTL),
Type: sp("TXT"),
},
},
}
var client route53Client
changes := client.computeChanges("n", testTree1, testTree0)
if !reflect.DeepEqual(changes, wantChanges) {
t.Fatalf("wrong changes (got %d, want %d)", len(changes), len(wantChanges))
}
// Check splitting according to size.
wantSplit := [][]*route53.Change{
wantChanges[:4],
wantChanges[4:6],
wantChanges[6:],
}
split := splitChanges(changes, 600, 4000)
if !reflect.DeepEqual(split, wantSplit) {
t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit))
}
// Check splitting according to count.
wantSplit = [][]*route53.Change{
wantChanges[:5],
wantChanges[5:],
}
split = splitChanges(changes, 10000, 6)
if !reflect.DeepEqual(split, wantSplit) {
t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit))
}
}
func sp(s string) *string { return &s }
func ip(i int64) *int64 { return &i }

5
cmd/devp2p/dnscmd.go
View file

@ -96,6 +96,11 @@ var (
}
)
const (
rootTTL = 30 * 60 // 30 min
treeNodeTTL = 4 * 7 * 24 * 60 * 60 // 4 weeks
)
// dnsSync performs dnsSyncCommand.
func dnsSync(ctx *cli.Context) error {
var (

3
cmd/devp2p/main.go
View file

@ -45,7 +45,7 @@ func init() {
// Set up the CLI app.
app.Flags = append(app.Flags, debug.Flags...)
app.Before = func(ctx *cli.Context) error {
return debug.Setup(ctx, "")
return debug.Setup(ctx)
}
app.After = func(ctx *cli.Context) error {
debug.Exit()
@ -59,6 +59,7 @@ func init() {
app.Commands = []cli.Command{
enrdumpCommand,
discv4Command,
discv5Command,
dnsCommand,
nodesetCommand,
}

2
cmd/devp2p/nodesetcmd.go
View file

@ -164,7 +164,7 @@ func ethFilter(args []string) (nodeFilter, error) {
case "goerli":
filter = forkid.NewStaticFilter(params.GoerliChainConfig, params.GoerliGenesisHash)
case "ropsten":
filter = forkid.NewStaticFilter(params.TestnetChainConfig, params.TestnetGenesisHash)
filter = forkid.NewStaticFilter(params.RopstenChainConfig, params.RopstenGenesisHash)
default:
return nil, fmt.Errorf("unknown network %q", args[0])
}

10
cmd/ethkey/generate.go
View file

@ -52,6 +52,10 @@ If you want to encrypt an existing private key, it can be specified by setting
Name: "privatekey",
Usage: "file containing a raw private key to encrypt",
},
cli.BoolFlag{
Name: "lightkdf",
Usage: "use less secure scrypt parameters",
},
},
Action: func(ctx *cli.Context) error {
// Check if keyfile path given and make sure it doesn't already exist.
@ -91,7 +95,11 @@ If you want to encrypt an existing private key, it can be specified by setting
// Encrypt key with passphrase.
passphrase := promptPassphrase(true)
keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP)
scryptN, scryptP := keystore.StandardScryptN, keystore.StandardScryptP
if ctx.Bool("lightkdf") {
scryptN, scryptP = keystore.LightScryptN, keystore.LightScryptP
}
keyjson, err := keystore.EncryptKey(key, passphrase, scryptN, scryptP)
if err != nil {
utils.Fatalf("Error encrypting key: %v", err)
}

2
cmd/ethkey/message_test.go
View file

@ -34,7 +34,7 @@ func TestMessageSignVerify(t *testing.T) {
message := "test message"
// Create the key.
generate := runEthkey(t, "generate", keyfile)
generate := runEthkey(t, "generate", "--lightkdf", keyfile)
generate.Expect(`
!! Unsupported terminal, password will be echoed.
Password: {{.InputLine "foobar"}}

23
cmd/evm/disasm.go
View file

@ -34,17 +34,22 @@ var disasmCommand = cli.Command{
}
func disasmCmd(ctx *cli.Context) error {
if len(ctx.Args().First()) == 0 {
return errors.New("filename required")
var in string
switch {
case len(ctx.Args().First()) > 0:
fn := ctx.Args().First()
input, err := ioutil.ReadFile(fn)
if err != nil {
return err
}
in = string(input)
case ctx.GlobalIsSet(InputFlag.Name):
in = ctx.GlobalString(InputFlag.Name)
default:
return errors.New("Missing filename or --input value")
}
fn := ctx.Args().First()
in, err := ioutil.ReadFile(fn)
if err != nil {
return err
}
code := strings.TrimSpace(string(in))
code := strings.TrimSpace(in)
fmt.Printf("%v\n", code)
return asm.PrintDisassembled(code)
}

50
cmd/evm/runner.go
View file

@ -70,14 +70,13 @@ func readGenesis(genesisPath string) *core.Genesis {
return genesis
}
func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, uint64, time.Duration, error) {
var (
output []byte
gasLeft uint64
execTime time.Duration
err error
)
type execStats struct {
time time.Duration // The execution time.
allocs int64 // The number of heap allocations during execution.
bytesAllocated int64 // The cumulative number of bytes allocated during execution.
}
func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, gasLeft uint64, stats execStats, err error) {
if bench {
result := testing.Benchmark(func(b *testing.B) {
for i := 0; i < b.N; i++ {
@ -87,14 +86,21 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, uin
// Get the average execution time from the benchmarking result.
// There are other useful stats here that could be reported.
execTime = time.Duration(result.NsPerOp())
stats.time = time.Duration(result.NsPerOp())
stats.allocs = result.AllocsPerOp()
stats.bytesAllocated = result.AllocedBytesPerOp()
} else {
var memStatsBefore, memStatsAfter goruntime.MemStats
goruntime.ReadMemStats(&memStatsBefore)
startTime := time.Now()
output, gasLeft, err = execFunc()
execTime = time.Since(startTime)
stats.time = time.Since(startTime)
goruntime.ReadMemStats(&memStatsAfter)
stats.allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs)
stats.bytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc)
}
return output, gasLeft, execTime, err
return output, gasLeft, stats, err
}
func runCmd(ctx *cli.Context) error {
@ -129,10 +135,10 @@ func runCmd(ctx *cli.Context) error {
genesisConfig = gen
db := rawdb.NewMemoryDatabase()
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil)
chainConfig = gen.Config
} else {
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
genesisConfig = new(core.Genesis)
}
if ctx.GlobalString(SenderFlag.Name) != "" {
@ -256,7 +262,8 @@ func runCmd(ctx *cli.Context) error {
}
}
output, leftOverGas, execTime, err := timedExec(ctx.GlobalBool(BenchFlag.Name), execFunc)
bench := ctx.GlobalBool(BenchFlag.Name)
output, leftOverGas, stats, err := timedExec(bench, execFunc)
if ctx.GlobalBool(DumpFlag.Name) {
statedb.Commit(true)
@ -286,17 +293,12 @@ func runCmd(ctx *cli.Context) error {
vm.WriteLogs(os.Stderr, statedb.Logs())
}
if ctx.GlobalBool(StatDumpFlag.Name) {
var mem goruntime.MemStats
goruntime.ReadMemStats(&mem)
fmt.Fprintf(os.Stderr, `evm execution time: %v
heap objects: %d
allocations: %d
total allocations: %d
GC calls: %d
Gas used: %d
`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
if bench || ctx.GlobalBool(StatDumpFlag.Name) {
fmt.Fprintf(os.Stderr, `EVM gas used: %d
execution time: %v
allocations: %d
allocated bytes: %d
`, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated)
}
if tracer == nil {
fmt.Printf("0x%x\n", output)

2
cmd/evm/staterunner.go
View file

@ -96,7 +96,7 @@ func stateTestCmd(ctx *cli.Context) error {
for _, st := range test.Subtests() {
// Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
state, err := test.Run(st, cfg)
state, err := test.Run(st, cfg, false)
// print state root for evmlab tracing
if ctx.GlobalBool(MachineFlag.Name) && state != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))

5
cmd/faucet/faucet.go
View file

@ -360,11 +360,14 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
}
}
// Send over the initial stats and the latest header
f.lock.RLock()
reqs := f.reqs
f.lock.RUnlock()
if err = send(conn, map[string]interface{}{
"funds": new(big.Int).Div(balance, ether),
"funded": nonce,
"peers": f.stack.Server().PeerCount(),
"requests": f.reqs,
"requests": reqs,
}, 3*time.Second); err != nil {
log.Warn("Failed to send initial stats to client", "err", err)
return

31
cmd/geth/accountcmd_test.go
View file

@ -88,6 +88,37 @@ Path of the secret key file: .*UTC--.+--[0-9a-f]{40}
`)
}
func TestAccountImport(t *testing.T) {
tests := []struct{ key, output string }{
{
key: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
output: "Address: {fcad0b19bb29d4674531d6f115237e16afce377c}\n",
},
{
key: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1",
output: "Fatal: Failed to load the private key: invalid character '1' at end of key file\n",
},
}
for _, test := range tests {
importAccountWithExpect(t, test.key, test.output)
}
}
func importAccountWithExpect(t *testing.T, key string, expected string) {
dir := tmpdir(t)
keyfile := filepath.Join(dir, "key.prv")
if err := ioutil.WriteFile(keyfile, []byte(key), 0600); err != nil {
t.Error(err)
}
passwordFile := filepath.Join(dir, "password.txt")
if err := ioutil.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil {
t.Error(err)
}
geth := runGeth(t, "account", "import", keyfile, "-password", passwordFile)
defer geth.ExpectExit()
geth.Expect(expected)
}
func TestAccountNewBadRepeat(t *testing.T) {
geth := runGeth(t, "account", "new", "--lightkdf")
defer geth.ExpectExit()

46
cmd/geth/chaincmd.go
View file

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/trie"
"gopkg.in/urfave/cli.v1"
)
@ -56,6 +57,18 @@ This is a destructive action and changes the network in which you will be
participating.
It expects the genesis file as argument.`,
}
dumpGenesisCommand = cli.Command{
Action: utils.MigrateFlags(dumpGenesis),
Name: "dumpgenesis",
Usage: "Dumps genesis block JSON configuration to stdout",
ArgsUsage: "",
Flags: []cli.Flag{
utils.DataDirFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
The dumpgenesis command dumps the genesis block configuration in JSON format to stdout.`,
}
importCommand = cli.Command{
Action: utils.MigrateFlags(importChain),
@ -67,8 +80,17 @@ It expects the genesis file as argument.`,
utils.CacheFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.SnapshotFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.MetricsEnabledFlag,
utils.MetricsEnabledExpensiveFlag,
utils.MetricsEnableInfluxDBFlag,
utils.MetricsInfluxDBEndpointFlag,
utils.MetricsInfluxDBDatabaseFlag,
utils.MetricsInfluxDBUsernameFlag,
utils.MetricsInfluxDBPasswordFlag,
utils.MetricsInfluxDBTagsFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -134,8 +156,10 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
utils.CacheFlag,
utils.SyncModeFlag,
utils.FakePoWFlag,
utils.TestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.LegacyTestnetFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -181,9 +205,10 @@ Use "ethereum dump 0" to dump the genesis block.`,
utils.DataDirFlag,
utils.AncientFlag,
utils.CacheFlag,
utils.TestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.LegacyTestnetFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
@ -227,10 +252,25 @@ func initGenesis(ctx *cli.Context) error {
return nil
}
func dumpGenesis(ctx *cli.Context) error {
genesis := utils.MakeGenesis(ctx)
if genesis == nil {
genesis = core.DefaultGenesisBlock()
}
if err := json.NewEncoder(os.Stdout).Encode(genesis); err != nil {
utils.Fatalf("could not encode genesis")
}
return nil
}
func importChain(ctx *cli.Context) error {
if len(ctx.Args()) < 1 {
utils.Fatalf("This command requires an argument.")
}
// Start metrics export if enabled
utils.SetupMetrics(ctx)
// Start system runtime metrics collection
go metrics.CollectProcessMetrics(3 * time.Second)
stack := makeFullNode(ctx)
defer stack.Close()
@ -521,7 +561,7 @@ func dump(ctx *cli.Context) error {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb))
state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}

11
cmd/geth/config.go
View file

@ -20,7 +20,6 @@ import (
"bufio"
"errors"
"fmt"
"math/big"
"os"
"reflect"
"unicode"
@ -99,8 +98,8 @@ func defaultNodeConfig() node.Config {
cfg := node.DefaultConfig
cfg.Name = clientIdentifier
cfg.Version = params.VersionWithCommit(gitCommit, gitDate)
cfg.HTTPModules = append(cfg.HTTPModules, "eth", "shh")
cfg.WSModules = append(cfg.WSModules, "eth", "shh")
cfg.HTTPModules = append(cfg.HTTPModules, "eth")
cfg.WSModules = append(cfg.WSModules, "eth")
cfg.IPCPath = "geth.ipc"
return cfg
}
@ -147,12 +146,6 @@ func enableWhisper(ctx *cli.Context) bool {
func makeFullNode(ctx *cli.Context) *node.Node {
stack, cfg := makeConfigNode(ctx)
if ctx.GlobalIsSet(utils.OverrideIstanbulFlag.Name) {
cfg.Eth.OverrideIstanbul = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideIstanbulFlag.Name))
}
if ctx.GlobalIsSet(utils.OverrideMuirGlacierFlag.Name) {
cfg.Eth.OverrideMuirGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideMuirGlacierFlag.Name))
}
utils.RegisterEthService(stack, &cfg.Eth)
// Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode

13
cmd/geth/consolecmd.go
View file

@ -123,10 +123,19 @@ func remoteConsole(ctx *cli.Context) error {
path = ctx.GlobalString(utils.DataDirFlag.Name)
}
if path != "" {
if ctx.GlobalBool(utils.TestnetFlag.Name) {
path = filepath.Join(path, "testnet")
if ctx.GlobalBool(utils.LegacyTestnetFlag.Name) || ctx.GlobalBool(utils.RopstenFlag.Name) {
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(path, "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
path = legacyPath
} else {
path = filepath.Join(path, "ropsten")
}
} else if ctx.GlobalBool(utils.RinkebyFlag.Name) {
path = filepath.Join(path, "rinkeby")
} else if ctx.GlobalBool(utils.GoerliFlag.Name) {
path = filepath.Join(path, "goerli")
}
}
endpoint = fmt.Sprintf("%s/geth.ipc", path)

8
cmd/geth/consolecmd_test.go
View file

@ -51,7 +51,9 @@ func TestConsoleWelcome(t *testing.T) {
geth.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
geth.SetTemplateFunc("gover", runtime.Version)
geth.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") })
geth.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
geth.SetTemplateFunc("niltime", func() string {
return time.Unix(0, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
})
geth.SetTemplateFunc("apis", func() string { return ipcAPIs })
// Verify the actual welcome message to the required template
@ -142,7 +144,9 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
attach.SetTemplateFunc("gover", runtime.Version)
attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") })
attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
attach.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
attach.SetTemplateFunc("niltime", func() string {
return time.Unix(0, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
})
attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
attach.SetTemplateFunc("datadir", func() string { return geth.Datadir })
attach.SetTemplateFunc("apis", func() string { return apis })

28
cmd/geth/genesis_test.go
View file

@ -28,22 +28,6 @@ var customGenesisTests = []struct {
query string
result string
}{
// Plain genesis file without anything extra
{
genesis: `{
"alloc" : {},
"coinbase" : "0x0000000000000000000000000000000000000000",
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00"
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
},
// Genesis file with an empty chain configuration (ensure missing fields work)
{
genesis: `{
@ -52,14 +36,14 @@ var customGenesisTests = []struct {
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"nonce" : "0x0000000000001338",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00",
"config" : {}
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
result: "0x0000000000001338",
},
// Genesis file with specific chain configurations
{
@ -69,7 +53,7 @@ var customGenesisTests = []struct {
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"nonce" : "0x0000000000001339",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00",
@ -80,7 +64,7 @@ var customGenesisTests = []struct {
}
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
result: "0x0000000000001339",
},
}
@ -97,10 +81,10 @@ func TestCustomGenesis(t *testing.T) {
if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
t.Fatalf("test %d: failed to write genesis file: %v", i, err)
}
runGeth(t, "--datadir", datadir, "init", json).WaitExit()
runGeth(t, "--nousb", "--datadir", datadir, "init", json).WaitExit()
// Query the custom genesis block
geth := runGeth(t,
geth := runGeth(t, "--nousb",
"--datadir", datadir, "--maxpeers", "0", "--port", "0",
"--nodiscover", "--nat", "none", "--ipcdisable",
"--exec", tt.query, "console")

37
cmd/geth/main.go
View file

@ -69,14 +69,14 @@ var (
utils.ExternalSignerFlag,
utils.NoUSBFlag,
utils.SmartCardDaemonPathFlag,
utils.OverrideIstanbulFlag,
utils.OverrideMuirGlacierFlag,
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,
utils.EthashDatasetsOnDiskFlag,
utils.EthashDatasetsLockMmapFlag,
utils.TxPoolLocalsFlag,
utils.TxPoolNoLocalsFlag,
utils.TxPoolJournalFlag,
@ -91,6 +91,7 @@ var (
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
utils.SnapshotFlag,
utils.LightServeFlag,
utils.LightLegacyServFlag,
utils.LightIngressFlag,
@ -106,6 +107,7 @@ var (
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.CacheSnapshotFlag,
utils.CacheNoPrefetchFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
@ -131,9 +133,11 @@ var (
utils.NetrestrictFlag,
utils.NodeKeyFileFlag,
utils.NodeKeyHexFlag,
utils.DNSDiscoveryFlag,
utils.DeveloperFlag,
utils.DeveloperPeriodFlag,
utils.TestnetFlag,
utils.LegacyTestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.VMEnableDebugFlag,
@ -205,6 +209,7 @@ func init() {
copydbCommand,
removedbCommand,
dumpCommand,
dumpGenesisCommand,
inspectCommand,
// See accountcmd.go:
accountCommand,
@ -233,7 +238,7 @@ func init() {
app.Flags = append(app.Flags, metricsFlags...)
app.Before = func(ctx *cli.Context) error {
return debug.Setup(ctx, "")
return debug.Setup(ctx)
}
app.After = func(ctx *cli.Context) error {
debug.Exit()
@ -252,10 +257,32 @@ func main() {
// prepare manipulates memory cache allowance and setups metric system.
// This function should be called before launching devp2p stack.
func prepare(ctx *cli.Context) {
// If we're running a known preset, log it for convenience.
switch {
case ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name):
log.Info("Starting Geth on Ropsten testnet...")
log.Warn("The --testnet flag is ambiguous! Please specify one of --goerli, --rinkeby, or --ropsten.")
log.Warn("The generic --testnet flag is deprecated and will be removed in the future!")
case ctx.GlobalIsSet(utils.RopstenFlag.Name):
log.Info("Starting Geth on Ropsten testnet...")
case ctx.GlobalIsSet(utils.RinkebyFlag.Name):
log.Info("Starting Geth on Rinkeby testnet...")
case ctx.GlobalIsSet(utils.GoerliFlag.Name):
log.Info("Starting Geth on Görli testnet...")
case ctx.GlobalIsSet(utils.DeveloperFlag.Name):
log.Info("Starting Geth in ephemeral dev mode...")
case !ctx.GlobalIsSet(utils.NetworkIdFlag.Name):
log.Info("Starting Geth on Ethereum mainnet...")
}
// If we're a full node on mainnet without --cache specified, bump default cache allowance
if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
// Make sure we're not on any supported preconfigured testnet either
if !ctx.GlobalIsSet(utils.TestnetFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
if !ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name) && !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
// Nope, we're really on mainnet. Bump that cache up!
log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))

72
cmd/geth/retesteth.go
View file

@ -80,6 +80,7 @@ type RetestethEthAPI interface {
SendRawTransaction(ctx context.Context, rawTx hexutil.Bytes) (common.Hash, error)
BlockNumber(ctx context.Context) (uint64, error)
GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error)
GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error)
GetBalance(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (*math.HexOrDecimal256, error)
GetCode(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (hexutil.Bytes, error)
GetTransactionCount(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (uint64, error)
@ -110,7 +111,6 @@ type RetestethAPI struct {
genesisHash common.Hash
engine *NoRewardEngine
blockchain *core.BlockChain
blockNumber uint64
txMap map[common.Address]map[uint64]*types.Transaction // Sender -> Nonce -> Transaction
txSenders map[common.Address]struct{} // Set of transaction senders
blockInterval uint64
@ -356,7 +356,7 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
ChainID: chainId,
HomesteadBlock: homesteadBlock,
DAOForkBlock: daoForkBlock,
DAOForkSupport: false,
DAOForkSupport: true,
EIP150Block: eip150Block,
EIP155Block: eip155Block,
EIP158Block: eip158Block,
@ -387,11 +387,13 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
inner = ethash.NewFaker()
case "Ethash":
inner = ethash.New(ethash.Config{
CacheDir: "ethash",
CachesInMem: 2,
CachesOnDisk: 3,
DatasetsInMem: 1,
DatasetsOnDisk: 2,
CacheDir: "ethash",
CachesInMem: 2,
CachesOnDisk: 3,
CachesLockMmap: false,
DatasetsInMem: 1,
DatasetsOnDisk: 2,
DatasetsLockMmap: false,
}, nil, false)
default:
return false, fmt.Errorf("unrecognised seal engine: %s", chainParams.SealEngine)
@ -411,7 +413,6 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
api.engine = engine
api.blockchain = blockchain
api.db = state.NewDatabase(api.ethDb)
api.blockNumber = 0
api.txMap = make(map[common.Address]map[uint64]*types.Transaction)
api.txSenders = make(map[common.Address]struct{})
api.blockInterval = 0
@ -424,7 +425,7 @@ func (api *RetestethAPI) SendRawTransaction(ctx context.Context, rawTx hexutil.B
// Return nil is not by mistake - some tests include sending transaction where gasLimit overflows uint64
return common.Hash{}, nil
}
signer := types.MakeSigner(api.chainConfig, big.NewInt(int64(api.blockNumber)))
signer := types.MakeSigner(api.chainConfig, big.NewInt(int64(api.currentNumber())))
sender, err := types.Sender(signer, tx)
if err != nil {
return common.Hash{}, err
@ -450,9 +451,17 @@ func (api *RetestethAPI) MineBlocks(ctx context.Context, number uint64) (bool, e
return true, nil
}
func (api *RetestethAPI) currentNumber() uint64 {
if current := api.blockchain.CurrentBlock(); current != nil {
return current.NumberU64()
}
return 0
}
func (api *RetestethAPI) mineBlock() error {
parentHash := rawdb.ReadCanonicalHash(api.ethDb, api.blockNumber)
parent := rawdb.ReadBlock(api.ethDb, parentHash, api.blockNumber)
number := api.currentNumber()
parentHash := rawdb.ReadCanonicalHash(api.ethDb, number)
parent := rawdb.ReadBlock(api.ethDb, parentHash, number)
var timestamp uint64
if api.blockInterval == 0 {
timestamp = uint64(time.Now().Unix())
@ -462,7 +471,7 @@ func (api *RetestethAPI) mineBlock() error {
gasLimit := core.CalcGasLimit(parent, 9223372036854775807, 9223372036854775807)
header := &types.Header{
ParentHash: parent.Hash(),
Number: big.NewInt(int64(api.blockNumber + 1)),
Number: big.NewInt(int64(number + 1)),
GasLimit: gasLimit,
Extra: api.extraData,
Time: timestamp,
@ -548,8 +557,7 @@ func (api *RetestethAPI) importBlock(block *types.Block) error {
if _, err := api.blockchain.InsertChain([]*types.Block{block}); err != nil {
return err
}
api.blockNumber = block.NumberU64()
fmt.Printf("Imported block %d\n", block.NumberU64())
fmt.Printf("Imported block %d, head is %d\n", block.NumberU64(), api.currentNumber())
return nil
}
@ -574,7 +582,9 @@ func (api *RetestethAPI) RewindToBlock(ctx context.Context, newHead uint64) (boo
if err := api.blockchain.SetHead(newHead); err != nil {
return false, err
}
api.blockNumber = newHead
// When we rewind, the transaction pool should be cleaned out.
api.txMap = make(map[common.Address]map[uint64]*types.Transaction)
api.txSenders = make(map[common.Address]struct{})
return true, nil
}
@ -594,8 +604,7 @@ func (api *RetestethAPI) GetLogHash(ctx context.Context, txHash common.Hash) (co
}
func (api *RetestethAPI) BlockNumber(ctx context.Context) (uint64, error) {
//fmt.Printf("BlockNumber, response: %d\n", api.blockNumber)
return api.blockNumber, nil
return api.currentNumber(), nil
}
func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error) {
@ -612,6 +621,20 @@ func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexO
return nil, fmt.Errorf("block %d not found", blockNr)
}
func (api *RetestethAPI) GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error) {
block := api.blockchain.GetBlockByHash(blockHash)
if block != nil {
response, err := RPCMarshalBlock(block, true, fullTx)
if err != nil {
return nil, err
}
response["author"] = response["miner"]
response["totalDifficulty"] = (*hexutil.Big)(api.blockchain.GetTd(block.Hash(), block.Number().Uint64()))
return response, err
}
return nil, fmt.Errorf("block 0x%x not found", blockHash)
}
func (api *RetestethAPI) AccountRange(ctx context.Context,
blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
addressHash *math.HexOrDecimal256, maxResults uint64,
@ -867,9 +890,22 @@ func retesteth(ctx *cli.Context) error {
vhosts := splitAndTrim(ctx.GlobalString(utils.RPCVirtualHostsFlag.Name))
cors := splitAndTrim(ctx.GlobalString(utils.RPCCORSDomainFlag.Name))
// register apis and create handler stack
srv := rpc.NewServer()
err := node.RegisterApisFromWhitelist(rpcAPI, []string{"test", "eth", "debug", "web3"}, srv, false)
if err != nil {
utils.Fatalf("Could not register RPC apis: %w", err)
}
handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
// start http server
var RetestethHTTPTimeouts = rpc.HTTPTimeouts{
ReadTimeout: 120 * time.Second,
WriteTimeout: 120 * time.Second,
IdleTimeout: 120 * time.Second,
}
httpEndpoint := fmt.Sprintf("%s:%d", ctx.GlobalString(utils.RPCListenAddrFlag.Name), ctx.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"test", "eth", "debug", "web3"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
listener, err := node.StartHTTPEndpoint(httpEndpoint, RetestethHTTPTimeouts, handler)
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}

9
cmd/geth/usage.go
View file

@ -72,9 +72,9 @@ var AppHelpFlagGroups = []flagGroup{
utils.NoUSBFlag,
utils.SmartCardDaemonPathFlag,
utils.NetworkIdFlag,
utils.TestnetFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.RinkebyFlag,
utils.RopstenFlag,
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
@ -109,9 +109,11 @@ var AppHelpFlagGroups = []flagGroup{
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,
utils.EthashDatasetsOnDiskFlag,
utils.EthashDatasetsLockMmapFlag,
},
},
{
@ -137,6 +139,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.CacheSnapshotFlag,
utils.CacheNoPrefetchFlag,
},
},
@ -182,6 +185,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.BootnodesFlag,
utils.BootnodesV4Flag,
utils.BootnodesV5Flag,
utils.DNSDiscoveryFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
utils.MaxPendingPeersFlag,
@ -241,6 +245,7 @@ var AppHelpFlagGroups = []flagGroup{
{
Name: "DEPRECATED",
Flags: []cli.Flag{
utils.LegacyTestnetFlag,
utils.LightLegacyServFlag,
utils.LightLegacyPeersFlag,
utils.MinerLegacyThreadsFlag,

4
cmd/puppeth/module_wallet.go
View file

@ -182,11 +182,11 @@ func checkWallet(client *sshClient, network string) (*walletInfos, error) {
// Run a sanity check to see if the devp2p and RPC ports are reachable
nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
if err = checkPort(client.server, nodePort); err != nil {
log.Warn(fmt.Sprintf("Wallet devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
log.Warn("Wallet devp2p port seems unreachable", "server", client.server, "port", nodePort, "err", err)
}
rpcPort := infos.portmap["8545/tcp"]
if err = checkPort(client.server, rpcPort); err != nil {
log.Warn(fmt.Sprintf("Wallet RPC port seems unreachable"), "server", client.server, "port", rpcPort, "err", err)
log.Warn("Wallet RPC port seems unreachable", "server", client.server, "port", rpcPort, "err", err)
}
// Assemble and return the useful infos
stats := &walletInfos{

2
cmd/utils/cmd.go
View file

@ -301,7 +301,7 @@ func ExportPreimages(db ethdb.Database, fn string) error {
defer writer.(*gzip.Writer).Close()
}
// Iterate over the preimages and export them
it := db.NewIteratorWithPrefix([]byte("secure-key-"))
it := db.NewIterator([]byte("secure-key-"), nil)
defer it.Release()
for it.Next() {

150
cmd/utils/flags.go
View file

@ -162,20 +162,24 @@ var (
}
NetworkIdFlag = cli.Uint64Flag{
Name: "networkid",
Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby)",
Usage: "Network identifier (integer, 1=Frontier, 3=Ropsten, 4=Rinkeby, 5=Görli)",
Value: eth.DefaultConfig.NetworkId,
}
TestnetFlag = cli.BoolFlag{
LegacyTestnetFlag = cli.BoolFlag{ // TODO(q9f): Remove after Ropsten is discontinued.
Name: "testnet",
Usage: "Ropsten network: pre-configured proof-of-work test network",
Usage: "Pre-configured test network (Deprecated: Please choose one of --goerli, --rinkeby, or --ropsten.)",
}
GoerliFlag = cli.BoolFlag{
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
}
RinkebyFlag = cli.BoolFlag{
Name: "rinkeby",
Usage: "Rinkeby network: pre-configured proof-of-authority test network",
}
GoerliFlag = cli.BoolFlag{
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
RopstenFlag = cli.BoolFlag{
Name: "ropsten",
Usage: "Ropsten network: pre-configured proof-of-work test network",
}
DeveloperFlag = cli.BoolFlag{
Name: "dev",
@ -225,6 +229,10 @@ var (
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
}
SnapshotFlag = cli.BoolFlag{
Name: "snapshot",
Usage: `Enables snapshot-database mode -- experimental work in progress feature`,
}
LightKDFFlag = cli.BoolFlag{
Name: "lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@ -233,14 +241,6 @@ var (
Name: "whitelist",
Usage: "Comma separated block number-to-hash mappings to enforce (<number>=<hash>)",
}
OverrideIstanbulFlag = cli.Uint64Flag{
Name: "override.istanbul",
Usage: "Manually specify Istanbul fork-block, overriding the bundled setting",
}
OverrideMuirGlacierFlag = cli.Uint64Flag{
Name: "override.muirglacier",
Usage: "Manually specify Muir Glacier fork-block, overriding the bundled setting",
}
// Light server and client settings
LightLegacyServFlag = cli.IntFlag{ // Deprecated in favor of light.serve, remove in 2021
Name: "lightserv",
@ -301,6 +301,10 @@ var (
Usage: "Number of recent ethash caches to keep on disk (16MB each)",
Value: eth.DefaultConfig.Ethash.CachesOnDisk,
}
EthashCachesLockMmapFlag = cli.BoolFlag{
Name: "ethash.cacheslockmmap",
Usage: "Lock memory maps of recent ethash caches",
}
EthashDatasetDirFlag = DirectoryFlag{
Name: "ethash.dagdir",
Usage: "Directory to store the ethash mining DAGs",
@ -316,6 +320,10 @@ var (
Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)",
Value: eth.DefaultConfig.Ethash.DatasetsOnDisk,
}
EthashDatasetsLockMmapFlag = cli.BoolFlag{
Name: "ethash.dagslockmmap",
Usage: "Lock memory maps for recent ethash mining DAGs",
}
// Transaction pool settings
TxPoolLocalsFlag = cli.StringFlag{
Name: "txpool.locals",
@ -383,14 +391,19 @@ var (
}
CacheTrieFlag = cli.IntFlag{
Name: "cache.trie",
Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)",
Value: 25,
Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)",
Value: 15,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
Value: 25,
}
CacheSnapshotFlag = cli.IntFlag{
Name: "cache.snapshot",
Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)",
Value: 10,
}
CacheNoPrefetchFlag = cli.BoolFlag{
Name: "cache.noprefetch",
Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)",
@ -658,6 +671,10 @@ var (
Name: "netrestrict",
Usage: "Restricts network communication to the given IP networks (CIDR masks)",
}
DNSDiscoveryFlag = cli.StringFlag{
Name: "discovery.dns",
Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)",
}
// ATM the url is left to the user and deployment to
JSpathFlag = cli.StringFlag{
@ -738,7 +755,6 @@ var (
Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements",
Value: "host=localhost",
}
EWASMInterpreterFlag = cli.StringFlag{
Name: "vm.ewasm",
Usage: "External ewasm configuration (default = built-in interpreter)",
@ -753,11 +769,17 @@ var (
// MakeDataDir retrieves the currently requested data directory, terminating
// if none (or the empty string) is specified. If the node is starting a testnet,
// the a subdirectory of the specified datadir will be used.
// then a subdirectory of the specified datadir will be used.
func MakeDataDir(ctx *cli.Context) string {
if path := ctx.GlobalString(DataDirFlag.Name); path != "" {
if ctx.GlobalBool(TestnetFlag.Name) {
return filepath.Join(path, "testnet")
if ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name) {
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(path, "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
return legacyPath
}
return filepath.Join(path, "ropsten")
}
if ctx.GlobalBool(RinkebyFlag.Name) {
return filepath.Join(path, "rinkeby")
@ -811,12 +833,12 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
switch {
case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV4Flag.Name):
if ctx.GlobalIsSet(BootnodesV4Flag.Name) {
urls = strings.Split(ctx.GlobalString(BootnodesV4Flag.Name), ",")
urls = splitAndTrim(ctx.GlobalString(BootnodesV4Flag.Name))
} else {
urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name))
}
case ctx.GlobalBool(TestnetFlag.Name):
urls = params.TestnetBootnodes
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
urls = params.RopstenBootnodes
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
@ -845,9 +867,9 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
switch {
case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV5Flag.Name):
if ctx.GlobalIsSet(BootnodesV5Flag.Name) {
urls = strings.Split(ctx.GlobalString(BootnodesV5Flag.Name), ",")
urls = splitAndTrim(ctx.GlobalString(BootnodesV5Flag.Name))
} else {
urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name))
}
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyBootnodes
@ -1219,8 +1241,16 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
case ctx.GlobalBool(DeveloperFlag.Name):
cfg.DataDir = "" // unless explicitly requested, use memory databases
case ctx.GlobalBool(TestnetFlag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
case (ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name)) && cfg.DataDir == node.DefaultDataDir():
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(node.DefaultDataDir(), "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
log.Warn("Using the deprecated `testnet` datadir. Future versions will store the Ropsten chain in `ropsten`.")
cfg.DataDir = legacyPath
} else {
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten")
}
case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir():
@ -1293,12 +1323,18 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) {
if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) {
cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
}
if ctx.GlobalIsSet(EthashCachesLockMmapFlag.Name) {
cfg.Ethash.CachesLockMmap = ctx.GlobalBool(EthashCachesLockMmapFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) {
cfg.Ethash.DatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) {
cfg.Ethash.DatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsLockMmapFlag.Name) {
cfg.Ethash.DatasetsLockMmap = ctx.GlobalBool(EthashDatasetsLockMmapFlag.Name)
}
}
func setMiner(ctx *cli.Context, cfg *miner.Config) {
@ -1414,7 +1450,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
CheckExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag)
CheckExclusive(ctx, LightLegacyServFlag, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
@ -1459,6 +1495,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
}
if !ctx.GlobalIsSet(SnapshotFlag.Name) {
cfg.SnapshotCache = 0 // Disabled
}
if ctx.GlobalIsSet(DocRootFlag.Name) {
cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)
}
@ -1477,24 +1519,35 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
if ctx.GlobalIsSet(RPCGlobalGasCap.Name) {
cfg.RPCGasCap = new(big.Int).SetUint64(ctx.GlobalUint64(RPCGlobalGasCap.Name))
}
if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) {
urls := ctx.GlobalString(DNSDiscoveryFlag.Name)
if urls == "" {
cfg.DiscoveryURLs = []string{}
} else {
cfg.DiscoveryURLs = splitAndTrim(urls)
}
}
// Override any default configs for hard coded networks.
switch {
case ctx.GlobalBool(TestnetFlag.Name):
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 3
}
cfg.Genesis = core.DefaultTestnetGenesisBlock()
cfg.Genesis = core.DefaultRopstenGenesisBlock()
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.RopstenGenesisHash])
case ctx.GlobalBool(RinkebyFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 4
}
cfg.Genesis = core.DefaultRinkebyGenesisBlock()
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.RinkebyGenesisHash])
case ctx.GlobalBool(GoerliFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 5
}
cfg.Genesis = core.DefaultGoerliGenesisBlock()
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.GoerliGenesisHash])
case ctx.GlobalBool(DeveloperFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 1337
@ -1521,9 +1574,22 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) {
cfg.Miner.GasPrice = big.NewInt(1)
}
default:
if cfg.NetworkId == 1 {
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.MainnetGenesisHash])
}
}
}
// setDNSDiscoveryDefaults configures DNS discovery with the given URL if
// no URLs are set.
func setDNSDiscoveryDefaults(cfg *eth.Config, url string) {
if cfg.DiscoveryURLs != nil {
return
}
cfg.DiscoveryURLs = []string{url}
}
// RegisterEthService adds an Ethereum client to the stack.
func RegisterEthService(stack *node.Node, cfg *eth.Config) {
var err error
@ -1651,8 +1717,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
func MakeGenesis(ctx *cli.Context) *core.Genesis {
var genesis *core.Genesis
switch {
case ctx.GlobalBool(TestnetFlag.Name):
genesis = core.DefaultTestnetGenesisBlock()
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
genesis = core.DefaultRopstenGenesisBlock()
case ctx.GlobalBool(RinkebyFlag.Name):
genesis = core.DefaultRinkebyGenesisBlock()
case ctx.GlobalBool(GoerliFlag.Name):
@ -1678,12 +1744,14 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
engine = ethash.NewFaker()
if !ctx.GlobalBool(FakePoWFlag.Name) {
engine = ethash.New(ethash.Config{
CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
CachesInMem: eth.DefaultConfig.Ethash.CachesInMem,
CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk,
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,
DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
CachesInMem: eth.DefaultConfig.Ethash.CachesInMem,
CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk,
CachesLockMmap: eth.DefaultConfig.Ethash.CachesLockMmap,
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,
DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
DatasetsLockMmap: eth.DefaultConfig.Ethash.DatasetsLockMmap,
}, nil, false)
}
}
@ -1696,6 +1764,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
SnapshotLimit: eth.DefaultConfig.SnapshotCache,
}
if !ctx.GlobalIsSet(SnapshotFlag.Name) {
cache.SnapshotLimit = 0 // Disabled
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100

1
cmd/wnode/main.go
View file

@ -599,6 +599,7 @@ func messageLoop() {
}
ticker := time.NewTicker(time.Millisecond * 50)
defer ticker.Stop()
for {
select {

11
common/bytes.go
View file

@ -145,3 +145,14 @@ func TrimLeftZeroes(s []byte) []byte {
}
return s[idx:]
}
// TrimRightZeroes returns a subslice of s without trailing zeroes
func TrimRightZeroes(s []byte) []byte {
idx := len(s)
for ; idx > 0; idx-- {
if s[idx-1] != 0 {
break
}
}
return s[:idx]
}

19
common/bytes_test.go
View file

@ -105,3 +105,22 @@ func TestNoPrefixShortHexOddLength(t *testing.T) {
t.Errorf("Expected %x got %x", expected, result)
}
}
func TestTrimRightZeroes(t *testing.T) {
tests := []struct {
arr []byte
exp []byte
}{
{FromHex("0x00ffff00ff0000"), FromHex("0x00ffff00ff")},
{FromHex("0x00000000000000"), []byte{}},
{FromHex("0xff"), FromHex("0xff")},
{[]byte{}, []byte{}},
{FromHex("0x00ffffffffffff"), FromHex("0x00ffffffffffff")},
}
for i, test := range tests {
got := TrimRightZeroes(test.arr)
if !bytes.Equal(got, test.exp) {
t.Errorf("test %d, got %x exp %x", i, got, test.exp)
}
}
}

65
common/mclock/mclock.go
View file

@ -31,44 +31,93 @@ func Now() AbsTime {
return AbsTime(monotime.Now())
}
// Add returns t + d.
// Add returns t + d as absolute time.
func (t AbsTime) Add(d time.Duration) AbsTime {
return t + AbsTime(d)
}
// Sub returns t - t2 as a duration.
func (t AbsTime) Sub(t2 AbsTime) time.Duration {
return time.Duration(t - t2)
}
// The Clock interface makes it possible to replace the monotonic system clock with
// a simulated clock.
type Clock interface {
Now() AbsTime
Sleep(time.Duration)
After(time.Duration) <-chan time.Time
NewTimer(time.Duration) ChanTimer
After(time.Duration) <-chan AbsTime
AfterFunc(d time.Duration, f func()) Timer
}
// Timer represents a cancellable event returned by AfterFunc
// Timer is a cancellable event created by AfterFunc.
type Timer interface {
// Stop cancels the timer. It returns false if the timer has already
// expired or been stopped.
Stop() bool
}
// ChanTimer is a cancellable event created by NewTimer.
type ChanTimer interface {
Timer
// The channel returned by C receives a value when the timer expires.
C() <-chan AbsTime
// Reset reschedules the timer with a new timeout.
// It should be invoked only on stopped or expired timers with drained channels.
Reset(time.Duration)
}
// System implements Clock using the system clock.
type System struct{}
// Now returns the current monotonic time.
func (System) Now() AbsTime {
func (c System) Now() AbsTime {
return AbsTime(monotime.Now())
}
// Sleep blocks for the given duration.
func (System) Sleep(d time.Duration) {
func (c System) Sleep(d time.Duration) {
time.Sleep(d)
}
// NewTimer creates a timer which can be rescheduled.
func (c System) NewTimer(d time.Duration) ChanTimer {
ch := make(chan AbsTime, 1)
t := time.AfterFunc(d, func() {
// This send is non-blocking because that's how time.Timer
// behaves. It doesn't matter in the happy case, but does
// when Reset is misused.
select {
case ch <- c.Now():
default:
}
})
return &systemTimer{t, ch}
}
// After returns a channel which receives the current time after d has elapsed.
func (System) After(d time.Duration) <-chan time.Time {
return time.After(d)
func (c System) After(d time.Duration) <-chan AbsTime {
ch := make(chan AbsTime, 1)
time.AfterFunc(d, func() { ch <- c.Now() })
return ch
}
// AfterFunc runs f on a new goroutine after the duration has elapsed.
func (System) AfterFunc(d time.Duration, f func()) Timer {
func (c System) AfterFunc(d time.Duration, f func()) Timer {
return time.AfterFunc(d, f)
}
type systemTimer struct {
*time.Timer
ch <-chan AbsTime
}
func (st *systemTimer) Reset(d time.Duration) {
st.Timer.Reset(d)
}
func (st *systemTimer) C() <-chan AbsTime {
return st.ch
}

143
common/mclock/simclock.go
View file

@ -17,6 +17,7 @@
package mclock
import (
"container/heap"
"sync"
"time"
)
@ -32,18 +33,24 @@ import (
// the timeout using a channel or semaphore.
type Simulated struct {
now AbsTime
scheduled []*simTimer
scheduled simTimerHeap
mu sync.RWMutex
cond *sync.Cond
lastId uint64
}
// simTimer implements Timer on the virtual clock.
// simTimer implements ChanTimer on the virtual clock.
type simTimer struct {
do func()
at AbsTime
id uint64
s *Simulated
at AbsTime
index int // position in s.scheduled
s *Simulated
do func()
ch <-chan AbsTime
}
func (s *Simulated) init() {
if s.cond == nil {
s.cond = sync.NewCond(&s.mu)
}
}
// Run moves the clock by the given duration, executing all timers before that duration.
@ -53,14 +60,9 @@ func (s *Simulated) Run(d time.Duration) {
end := s.now + AbsTime(d)
var do []func()
for len(s.scheduled) > 0 {
ev := s.scheduled[0]
if ev.at > end {
break
}
s.now = ev.at
for len(s.scheduled) > 0 && s.scheduled[0].at <= end {
ev := heap.Pop(&s.scheduled).(*simTimer)
do = append(do, ev.do)
s.scheduled = s.scheduled[1:]
}
s.now = end
s.mu.Unlock()
@ -102,14 +104,22 @@ func (s *Simulated) Sleep(d time.Duration) {
<-s.After(d)
}
// NewTimer creates a timer which fires when the clock has advanced by d.
func (s *Simulated) NewTimer(d time.Duration) ChanTimer {
s.mu.Lock()
defer s.mu.Unlock()
ch := make(chan AbsTime, 1)
var timer *simTimer
timer = s.schedule(d, func() { ch <- timer.at })
timer.ch = ch
return timer
}
// After returns a channel which receives the current time after the clock
// has advanced by d.
func (s *Simulated) After(d time.Duration) <-chan time.Time {
after := make(chan time.Time, 1)
s.AfterFunc(d, func() {
after <- (time.Time{}).Add(time.Duration(s.now))
})
return after
func (s *Simulated) After(d time.Duration) <-chan AbsTime {
return s.NewTimer(d).C()
}
// AfterFunc runs fn after the clock has advanced by d. Unlike with the system
@ -117,46 +127,83 @@ func (s *Simulated) After(d time.Duration) <-chan time.Time {
func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer {
s.mu.Lock()
defer s.mu.Unlock()
return s.schedule(d, fn)
}
func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer {
s.init()
at := s.now + AbsTime(d)
s.lastId++
id := s.lastId
l, h := 0, len(s.scheduled)
ll := h
for l != h {
m := (l + h) / 2
if (at < s.scheduled[m].at) || ((at == s.scheduled[m].at) && (id < s.scheduled[m].id)) {
h = m
} else {
l = m + 1
}
}
ev := &simTimer{do: fn, at: at, s: s}
s.scheduled = append(s.scheduled, nil)
copy(s.scheduled[l+1:], s.scheduled[l:ll])
s.scheduled[l] = ev
heap.Push(&s.scheduled, ev)
s.cond.Broadcast()
return ev
}
func (ev *simTimer) Stop() bool {
s := ev.s
s.mu.Lock()
defer s.mu.Unlock()
ev.s.mu.Lock()
defer ev.s.mu.Unlock()
for i := 0; i < len(s.scheduled); i++ {
if s.scheduled[i] == ev {
s.scheduled = append(s.scheduled[:i], s.scheduled[i+1:]...)
s.cond.Broadcast()
return true
}
if ev.index < 0 {
return false
}
return false
heap.Remove(&ev.s.scheduled, ev.index)
ev.s.cond.Broadcast()
ev.index = -1
return true
}
func (s *Simulated) init() {
if s.cond == nil {
s.cond = sync.NewCond(&s.mu)
func (ev *simTimer) Reset(d time.Duration) {
if ev.ch == nil {
panic("mclock: Reset() on timer created by AfterFunc")
}
ev.s.mu.Lock()
defer ev.s.mu.Unlock()
ev.at = ev.s.now.Add(d)
if ev.index < 0 {
heap.Push(&ev.s.scheduled, ev) // already expired
} else {
heap.Fix(&ev.s.scheduled, ev.index) // hasn't fired yet, reschedule
}
ev.s.cond.Broadcast()
}
func (ev *simTimer) C() <-chan AbsTime {
if ev.ch == nil {
panic("mclock: C() on timer created by AfterFunc")
}
return ev.ch
}
type simTimerHeap []*simTimer
func (h *simTimerHeap) Len() int {
return len(*h)
}
func (h *simTimerHeap) Less(i, j int) bool {
return (*h)[i].at < (*h)[j].at
}
func (h *simTimerHeap) Swap(i, j int) {
(*h)[i], (*h)[j] = (*h)[j], (*h)[i]
(*h)[i].index = i
(*h)[j].index = j
}
func (h *simTimerHeap) Push(x interface{}) {
t := x.(*simTimer)
t.index = len(*h)
*h = append(*h, t)
}
func (h *simTimerHeap) Pop() interface{} {
end := len(*h) - 1
t := (*h)[end]
t.index = -1
(*h)[end] = nil
*h = (*h)[:end]
return t
}

65
common/mclock/simclock_test.go
View file

@ -25,14 +25,16 @@ var _ Clock = System{}
var _ Clock = new(Simulated)
func TestSimulatedAfter(t *testing.T) {
const timeout = 30 * time.Minute
const adv = time.Minute
var (
c Simulated
end = c.Now().Add(timeout)
ch = c.After(timeout)
timeout = 30 * time.Minute
offset = 99 * time.Hour
adv = 11 * time.Minute
c Simulated
)
c.Run(offset)
end := c.Now().Add(timeout)
ch := c.After(timeout)
for c.Now() < end.Add(-adv) {
c.Run(adv)
select {
@ -45,8 +47,8 @@ func TestSimulatedAfter(t *testing.T) {
c.Run(adv)
select {
case stamp := <-ch:
want := time.Time{}.Add(timeout)
if !stamp.Equal(want) {
want := AbsTime(0).Add(offset).Add(timeout)
if stamp != want {
t.Errorf("Wrong time sent on timer channel: got %v, want %v", stamp, want)
}
default:
@ -94,7 +96,7 @@ func TestSimulatedSleep(t *testing.T) {
var (
c Simulated
timeout = 1 * time.Hour
done = make(chan AbsTime)
done = make(chan AbsTime, 1)
)
go func() {
c.Sleep(timeout)
@ -113,3 +115,48 @@ func TestSimulatedSleep(t *testing.T) {
t.Fatal("Sleep didn't return in time")
}
}
func TestSimulatedTimerReset(t *testing.T) {
var (
c Simulated
timeout = 1 * time.Hour
)
timer := c.NewTimer(timeout)
c.Run(2 * timeout)
select {
case ftime := <-timer.C():
if ftime != AbsTime(timeout) {
t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(timeout))
}
default:
t.Fatal("timer didn't fire")
}
timer.Reset(timeout)
c.Run(2 * timeout)
select {
case ftime := <-timer.C():
if ftime != AbsTime(3*timeout) {
t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(3*timeout))
}
default:
t.Fatal("timer didn't fire again")
}
}
func TestSimulatedTimerStop(t *testing.T) {
var (
c Simulated
timeout = 1 * time.Hour
)
timer := c.NewTimer(timeout)
c.Run(2 * timeout)
if timer.Stop() {
t.Errorf("Stop returned true for fired timer")
}
select {
case <-timer.C():
default:
t.Fatal("timer didn't fire")
}
}

19
common/prque/lazyqueue_test.go
View file

@ -74,17 +74,22 @@ func TestLazyQueue(t *testing.T) {
q.Push(&items[i])
}
var lock sync.Mutex
stopCh := make(chan chan struct{})
var (
lock sync.Mutex
wg sync.WaitGroup
stopCh = make(chan chan struct{})
)
defer wg.Wait()
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-clock.After(testQueueRefresh):
lock.Lock()
q.Refresh()
lock.Unlock()
case stop := <-stopCh:
close(stop)
case <-stopCh:
return
}
}
@ -104,6 +109,8 @@ func TestLazyQueue(t *testing.T) {
if rand.Intn(100) == 0 {
p := q.PopItem().(*lazyItem)
if p.p != maxPri {
lock.Unlock()
close(stopCh)
t.Fatalf("incorrect item (best known priority %d, popped %d)", maxPri, p.p)
}
q.Push(p)
@ -113,7 +120,5 @@ func TestLazyQueue(t *testing.T) {
clock.WaitForTimers(1)
}
stop := make(chan struct{})
stopCh <- stop
<-stop
close(stopCh)
}

27
consensus/ethash/algorithm_test.go
View file

@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
go func(idx int) {
defer pend.Done()
ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal, nil}, nil, false)
ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
defer ethash.Close()
if err := ethash.VerifySeal(nil, block.Header()); err != nil {
t.Errorf("proc %d: block verification failed: %v", idx, err)
@ -787,3 +787,28 @@ func BenchmarkHashimotoFullSmall(b *testing.B) {
hashimotoFull(dataset, hash, 0)
}
}
func benchmarkHashimotoFullMmap(b *testing.B, name string, lock bool) {
b.Run(name, func(b *testing.B) {
tmpdir, err := ioutil.TempDir("", "ethash-test")
if err != nil {
b.Fatal(err)
}
defer os.RemoveAll(tmpdir)
d := &dataset{epoch: 0}
d.generate(tmpdir, 1, lock, false)
var hash [common.HashLength]byte
b.ResetTimer()
for i := 0; i < b.N; i++ {
binary.PutVarint(hash[:], int64(i))
hashimotoFull(d.dataset, hash[:], 0)
}
})
}
// Benchmarks the full verification performance for mmap
func BenchmarkHashimotoFullMmap(b *testing.B) {
benchmarkHashimotoFullMmap(b, "WithLock", true)
benchmarkHashimotoFullMmap(b, "WithoutLock", false)
}

59
consensus/ethash/ethash.go
View file

@ -48,7 +48,7 @@ var (
two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
// sharedEthash is a full instance that can be shared between multiple users.
sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal, nil}, nil, false)
sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false)
// algorithmRevision is the data structure version used for file naming.
algorithmRevision = 23
@ -65,7 +65,7 @@ func isLittleEndian() bool {
}
// memoryMap tries to memory map a file of uint32s for read only access.
func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) {
func memoryMap(path string, lock bool) (*os.File, mmap.MMap, []uint32, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0644)
if err != nil {
return nil, nil, nil, err
@ -82,6 +82,13 @@ func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) {
return nil, nil, nil, ErrInvalidDumpMagic
}
}
if lock {
if err := mem.Lock(); err != nil {
mem.Unmap()
file.Close()
return nil, nil, nil, err
}
}
return file, mem, buffer[len(dumpMagic):], err
}
@ -107,7 +114,7 @@ func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) {
// memoryMapAndGenerate tries to memory map a temporary file of uint32s for write
// access, fill it with the data from a generator and then move it into the final
// path requested.
func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) {
func memoryMapAndGenerate(path string, size uint64, lock bool, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) {
// Ensure the data folder exists
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return nil, nil, nil, err
@ -142,7 +149,7 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint
if err := os.Rename(temp, path); err != nil {
return nil, nil, nil, err
}
return memoryMap(path)
return memoryMap(path, lock)
}
// lru tracks caches or datasets by their last use time, keeping at most N of them.
@ -213,7 +220,7 @@ func newCache(epoch uint64) interface{} {
}
// generate ensures that the cache content is generated before use.
func (c *cache) generate(dir string, limit int, test bool) {
func (c *cache) generate(dir string, limit int, lock bool, test bool) {
c.once.Do(func() {
size := cacheSize(c.epoch*epochLength + 1)
seed := seedHash(c.epoch*epochLength + 1)
@ -240,7 +247,7 @@ func (c *cache) generate(dir string, limit int, test bool) {
// Try to load the file from disk and memory map it
var err error
c.dump, c.mmap, c.cache, err = memoryMap(path)
c.dump, c.mmap, c.cache, err = memoryMap(path, lock)
if err == nil {
logger.Debug("Loaded old ethash cache from disk")
return
@ -248,7 +255,7 @@ func (c *cache) generate(dir string, limit int, test bool) {
logger.Debug("Failed to load old ethash cache", "err", err)
// No previous cache available, create a new cache file to fill
c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) })
c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, lock, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) })
if err != nil {
logger.Error("Failed to generate mapped ethash cache", "err", err)
@ -290,7 +297,7 @@ func newDataset(epoch uint64) interface{} {
}
// generate ensures that the dataset content is generated before use.
func (d *dataset) generate(dir string, limit int, test bool) {
func (d *dataset) generate(dir string, limit int, lock bool, test bool) {
d.once.Do(func() {
// Mark the dataset generated after we're done. This is needed for remote
defer atomic.StoreUint32(&d.done, 1)
@ -326,7 +333,7 @@ func (d *dataset) generate(dir string, limit int, test bool) {
// Try to load the file from disk and memory map it
var err error
d.dump, d.mmap, d.dataset, err = memoryMap(path)
d.dump, d.mmap, d.dataset, err = memoryMap(path, lock)
if err == nil {
logger.Debug("Loaded old ethash dataset from disk")
return
@ -337,7 +344,7 @@ func (d *dataset) generate(dir string, limit int, test bool) {
cache := make([]uint32, csize/4)
generateCache(cache, d.epoch, seed)
d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) })
d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, lock, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) })
if err != nil {
logger.Error("Failed to generate mapped ethash dataset", "err", err)
@ -372,13 +379,13 @@ func (d *dataset) finalizer() {
// MakeCache generates a new ethash cache and optionally stores it to disk.
func MakeCache(block uint64, dir string) {
c := cache{epoch: block / epochLength}
c.generate(dir, math.MaxInt32, false)
c.generate(dir, math.MaxInt32, false, false)
}
// MakeDataset generates a new ethash dataset and optionally stores it to disk.
func MakeDataset(block uint64, dir string) {
d := dataset{epoch: block / epochLength}
d.generate(dir, math.MaxInt32, false)
d.generate(dir, math.MaxInt32, false, false)
}
// Mode defines the type and amount of PoW verification an ethash engine makes.
@ -394,13 +401,15 @@ const (
// Config are the configuration parameters of the ethash.
type Config struct {
CacheDir string
CachesInMem int
CachesOnDisk int
DatasetDir string
DatasetsInMem int
DatasetsOnDisk int
PowMode Mode
CacheDir string
CachesInMem int
CachesOnDisk int
CachesLockMmap bool
DatasetDir string
DatasetsInMem int
DatasetsOnDisk int
DatasetsLockMmap bool
PowMode Mode
Log log.Logger `toml:"-"`
}
@ -549,12 +558,12 @@ func (ethash *Ethash) cache(block uint64) *cache {
current := currentI.(*cache)
// Wait for generation finish.
current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
// If we need a new future cache, now's a good time to regenerate it.
if futureI != nil {
future := futureI.(*cache)
go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
}
return current
}
@ -574,20 +583,20 @@ func (ethash *Ethash) dataset(block uint64, async bool) *dataset {
// If async is specified, generate everything in a background thread
if async && !current.generated() {
go func() {
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
if futureI != nil {
future := futureI.(*dataset)
future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
}
}()
} else {
// Either blocking generation was requested, or already done
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
if futureI != nil {
future := futureI.(*dataset)
go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
}
}
return current

379
console/bridge.go
View file

@ -20,14 +20,16 @@ import (
"encoding/json"
"fmt"
"io"
"reflect"
"strings"
"time"
"github.com/dop251/goja"
"github.com/ethereum/go-ethereum/accounts/scwallet"
"github.com/ethereum/go-ethereum/accounts/usbwallet"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/internal/jsre"
"github.com/ethereum/go-ethereum/rpc"
"github.com/robertkrimen/otto"
)
// bridge is a collection of JavaScript utility methods to bride the .js runtime
@ -47,10 +49,18 @@ func newBridge(client *rpc.Client, prompter UserPrompter, printer io.Writer) *br
}
}
func getJeth(vm *goja.Runtime) *goja.Object {
jeth := vm.Get("jeth")
if jeth == nil {
panic(vm.ToValue("jeth object does not exist"))
}
return jeth.ToObject(vm)
}
// NewAccount is a wrapper around the personal.newAccount RPC method that uses a
// non-echoing password prompt to acquire the passphrase and executes the original
// RPC method (saved in jeth.newAccount) with it to actually execute the RPC call.
func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) {
func (b *bridge) NewAccount(call jsre.Call) (goja.Value, error) {
var (
password string
confirm string
@ -58,52 +68,57 @@ func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) {
)
switch {
// No password was specified, prompt the user for it
case len(call.ArgumentList) == 0:
if password, err = b.prompter.PromptPassword("Password: "); err != nil {
throwJSException(err.Error())
case len(call.Arguments) == 0:
if password, err = b.prompter.PromptPassword("Passphrase: "); err != nil {
return nil, err
}
if confirm, err = b.prompter.PromptPassword("Repeat password: "); err != nil {
throwJSException(err.Error())
if confirm, err = b.prompter.PromptPassword("Repeat passphrase: "); err != nil {
return nil, err
}
if password != confirm {
throwJSException("passwords don't match!")
return nil, fmt.Errorf("passwords don't match!")
}
// A single string password was specified, use that
case len(call.ArgumentList) == 1 && call.Argument(0).IsString():
password, _ = call.Argument(0).ToString()
// Otherwise fail with some error
case len(call.Arguments) == 1 && call.Argument(0).ToString() != nil:
password = call.Argument(0).ToString().String()
default:
throwJSException("expected 0 or 1 string argument")
return nil, fmt.Errorf("expected 0 or 1 string argument")
}
// Password acquired, execute the call and return
ret, err := call.Otto.Call("jeth.newAccount", nil, password)
if err != nil {
throwJSException(err.Error())
newAccount, callable := goja.AssertFunction(getJeth(call.VM).Get("newAccount"))
if !callable {
return nil, fmt.Errorf("jeth.newAccount is not callable")
}
return ret
ret, err := newAccount(goja.Null(), call.VM.ToValue(password))
if err != nil {
return nil, err
}
return ret, nil
}
// OpenWallet is a wrapper around personal.openWallet which can interpret and
// react to certain error messages, such as the Trezor PIN matrix request.
func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) {
func (b *bridge) OpenWallet(call jsre.Call) (goja.Value, error) {
// Make sure we have a wallet specified to open
if !call.Argument(0).IsString() {
throwJSException("first argument must be the wallet URL to open")
if call.Argument(0).ToObject(call.VM).ClassName() != "String" {
return nil, fmt.Errorf("first argument must be the wallet URL to open")
}
wallet := call.Argument(0)
var passwd otto.Value
if call.Argument(1).IsUndefined() || call.Argument(1).IsNull() {
passwd, _ = otto.ToValue("")
var passwd goja.Value
if goja.IsUndefined(call.Argument(1)) || goja.IsNull(call.Argument(1)) {
passwd = call.VM.ToValue("")
} else {
passwd = call.Argument(1)
}
// Open the wallet and return if successful in itself
val, err := call.Otto.Call("jeth.openWallet", nil, wallet, passwd)
openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet"))
if !callable {
return nil, fmt.Errorf("jeth.openWallet is not callable")
}
val, err := openWallet(goja.Null(), wallet, passwd)
if err == nil {
return val
return val, nil
}
// Wallet open failed, report error unless it's a PIN or PUK entry
@ -111,32 +126,31 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) {
case strings.HasSuffix(err.Error(), usbwallet.ErrTrezorPINNeeded.Error()):
val, err = b.readPinAndReopenWallet(call)
if err == nil {
return val
return val, nil
}
val, err = b.readPassphraseAndReopenWallet(call)
if err != nil {
throwJSException(err.Error())
return nil, err
}
case strings.HasSuffix(err.Error(), scwallet.ErrPairingPasswordNeeded.Error()):
// PUK input requested, fetch from the user and call open again
if input, err := b.prompter.PromptPassword("Please enter the pairing password: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Please enter the pairing password: ")
if err != nil {
return nil, err
}
if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil {
passwd = call.VM.ToValue(input)
if val, err = openWallet(goja.Null(), wallet, passwd); err != nil {
if !strings.HasSuffix(err.Error(), scwallet.ErrPINNeeded.Error()) {
throwJSException(err.Error())
return nil, err
} else {
// PIN input requested, fetch from the user and call open again
if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Please enter current PIN: ")
if err != nil {
return nil, err
}
if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil {
throwJSException(err.Error())
if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(input)); err != nil {
return nil, err
}
}
}
@ -144,52 +158,52 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) {
case strings.HasSuffix(err.Error(), scwallet.ErrPINUnblockNeeded.Error()):
// PIN unblock requested, fetch PUK and new PIN from the user
var pukpin string
if input, err := b.prompter.PromptPassword("Please enter current PUK: "); err != nil {
throwJSException(err.Error())
} else {
pukpin = input
input, err := b.prompter.PromptPassword("Please enter current PUK: ")
if err != nil {
return nil, err
}
if input, err := b.prompter.PromptPassword("Please enter new PIN: "); err != nil {
throwJSException(err.Error())
} else {
pukpin += input
pukpin = input
input, err = b.prompter.PromptPassword("Please enter new PIN: ")
if err != nil {
return nil, err
}
passwd, _ = otto.ToValue(pukpin)
if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil {
throwJSException(err.Error())
pukpin += input
if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(pukpin)); err != nil {
return nil, err
}
case strings.HasSuffix(err.Error(), scwallet.ErrPINNeeded.Error()):
// PIN input requested, fetch from the user and call open again
if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Please enter current PIN: ")
if err != nil {
return nil, err
}
if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil {
throwJSException(err.Error())
if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(input)); err != nil {
return nil, err
}
default:
// Unknown error occurred, drop to the user
throwJSException(err.Error())
return nil, err
}
return val
return val, nil
}
func (b *bridge) readPassphraseAndReopenWallet(call otto.FunctionCall) (otto.Value, error) {
var passwd otto.Value
func (b *bridge) readPassphraseAndReopenWallet(call jsre.Call) (goja.Value, error) {
wallet := call.Argument(0)
if input, err := b.prompter.PromptPassword("Please enter your password: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Please enter your passphrase: ")
if err != nil {
return nil, err
}
return call.Otto.Call("jeth.openWallet", nil, wallet, passwd)
openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet"))
if !callable {
return nil, fmt.Errorf("jeth.openWallet is not callable")
}
return openWallet(goja.Null(), wallet, call.VM.ToValue(input))
}
func (b *bridge) readPinAndReopenWallet(call otto.FunctionCall) (otto.Value, error) {
var passwd otto.Value
func (b *bridge) readPinAndReopenWallet(call jsre.Call) (goja.Value, error) {
wallet := call.Argument(0)
// Trezor PIN matrix input requested, display the matrix to the user and fetch the data
fmt.Fprintf(b.printer, "Look at the device for number positions\n\n")
@ -199,155 +213,154 @@ func (b *bridge) readPinAndReopenWallet(call otto.FunctionCall) (otto.Value, err
fmt.Fprintf(b.printer, "--+---+--\n")
fmt.Fprintf(b.printer, "1 | 2 | 3\n\n")
if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Please enter current PIN: ")
if err != nil {
return nil, err
}
return call.Otto.Call("jeth.openWallet", nil, wallet, passwd)
openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet"))
if !callable {
return nil, fmt.Errorf("jeth.openWallet is not callable")
}
return openWallet(goja.Null(), wallet, call.VM.ToValue(input))
}
// UnlockAccount is a wrapper around the personal.unlockAccount RPC method that
// uses a non-echoing password prompt to acquire the passphrase and executes the
// original RPC method (saved in jeth.unlockAccount) with it to actually execute
// the RPC call.
func (b *bridge) UnlockAccount(call otto.FunctionCall) (response otto.Value) {
// Make sure we have an account specified to unlock
if !call.Argument(0).IsString() {
throwJSException("first argument must be the account to unlock")
func (b *bridge) UnlockAccount(call jsre.Call) (goja.Value, error) {
// Make sure we have an account specified to unlock.
if call.Argument(0).ExportType().Kind() != reflect.String {
return nil, fmt.Errorf("first argument must be the account to unlock")
}
account := call.Argument(0)
// If password is not given or is the null value, prompt the user for it
var passwd otto.Value
if call.Argument(1).IsUndefined() || call.Argument(1).IsNull() {
// If password is not given or is the null value, prompt the user for it.
var passwd goja.Value
if goja.IsUndefined(call.Argument(1)) || goja.IsNull(call.Argument(1)) {
fmt.Fprintf(b.printer, "Unlock account %s\n", account)
if input, err := b.prompter.PromptPassword("Password: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Passphrase: ")
if err != nil {
return nil, err
}
passwd = call.VM.ToValue(input)
} else {
if !call.Argument(1).IsString() {
throwJSException("password must be a string")
if call.Argument(1).ExportType().Kind() != reflect.String {
return nil, fmt.Errorf("password must be a string")
}
passwd = call.Argument(1)
}
// Third argument is the duration how long the account must be unlocked.
duration := otto.NullValue()
if call.Argument(2).IsDefined() && !call.Argument(2).IsNull() {
if !call.Argument(2).IsNumber() {
throwJSException("unlock duration must be a number")
// Third argument is the duration how long the account should be unlocked.
duration := goja.Null()
if !goja.IsUndefined(call.Argument(2)) && !goja.IsNull(call.Argument(2)) {
if !isNumber(call.Argument(2)) {
return nil, fmt.Errorf("unlock duration must be a number")
}
duration = call.Argument(2)
}
// Send the request to the backend and return
val, err := call.Otto.Call("jeth.unlockAccount", nil, account, passwd, duration)
if err != nil {
throwJSException(err.Error())
// Send the request to the backend and return.
unlockAccount, callable := goja.AssertFunction(getJeth(call.VM).Get("unlockAccount"))
if !callable {
return nil, fmt.Errorf("jeth.unlockAccount is not callable")
}
return val
return unlockAccount(goja.Null(), account, passwd, duration)
}
// Sign is a wrapper around the personal.sign RPC method that uses a non-echoing password
// prompt to acquire the passphrase and executes the original RPC method (saved in
// jeth.sign) with it to actually execute the RPC call.
func (b *bridge) Sign(call otto.FunctionCall) (response otto.Value) {
func (b *bridge) Sign(call jsre.Call) (goja.Value, error) {
var (
message = call.Argument(0)
account = call.Argument(1)
passwd = call.Argument(2)
)
if !message.IsString() {
throwJSException("first argument must be the message to sign")
if message.ExportType().Kind() != reflect.String {
return nil, fmt.Errorf("first argument must be the message to sign")
}
if !account.IsString() {
throwJSException("second argument must be the account to sign with")
if account.ExportType().Kind() != reflect.String {
return nil, fmt.Errorf("second argument must be the account to sign with")
}
// if the password is not given or null ask the user and ensure password is a string
if passwd.IsUndefined() || passwd.IsNull() {
if goja.IsUndefined(passwd) || goja.IsNull(passwd) {
fmt.Fprintf(b.printer, "Give password for account %s\n", account)
if input, err := b.prompter.PromptPassword("Password: "); err != nil {
throwJSException(err.Error())
} else {
passwd, _ = otto.ToValue(input)
input, err := b.prompter.PromptPassword("Password: ")
if err != nil {
return nil, err
}
}
if !passwd.IsString() {
throwJSException("third argument must be the password to unlock the account")
passwd = call.VM.ToValue(input)
} else if passwd.ExportType().Kind() != reflect.String {
return nil, fmt.Errorf("third argument must be the password to unlock the account")
}
// Send the request to the backend and return
val, err := call.Otto.Call("jeth.sign", nil, message, account, passwd)
if err != nil {
throwJSException(err.Error())
sign, callable := goja.AssertFunction(getJeth(call.VM).Get("unlockAccount"))
if !callable {
return nil, fmt.Errorf("jeth.unlockAccount is not callable")
}
return val
return sign(goja.Null(), message, account, passwd)
}
// Sleep will block the console for the specified number of seconds.
func (b *bridge) Sleep(call otto.FunctionCall) (response otto.Value) {
if call.Argument(0).IsNumber() {
sleep, _ := call.Argument(0).ToInteger()
time.Sleep(time.Duration(sleep) * time.Second)
return otto.TrueValue()
func (b *bridge) Sleep(call jsre.Call) (goja.Value, error) {
if !isNumber(call.Argument(0)) {
return nil, fmt.Errorf("usage: sleep(<number of seconds>)")
}
return throwJSException("usage: sleep(<number of seconds>)")
sleep := call.Argument(0).ToFloat()
time.Sleep(time.Duration(sleep * float64(time.Second)))
return call.VM.ToValue(true), nil
}
// SleepBlocks will block the console for a specified number of new blocks optionally
// until the given timeout is reached.
func (b *bridge) SleepBlocks(call otto.FunctionCall) (response otto.Value) {
func (b *bridge) SleepBlocks(call jsre.Call) (goja.Value, error) {
// Parse the input parameters for the sleep.
var (
blocks = int64(0)
sleep = int64(9999999999999999) // indefinitely
)
// Parse the input parameters for the sleep
nArgs := len(call.ArgumentList)
nArgs := len(call.Arguments)
if nArgs == 0 {
throwJSException("usage: sleepBlocks(<n blocks>[, max sleep in seconds])")
return nil, fmt.Errorf("usage: sleepBlocks(<n blocks>[, max sleep in seconds])")
}
if nArgs >= 1 {
if call.Argument(0).IsNumber() {
blocks, _ = call.Argument(0).ToInteger()
} else {
throwJSException("expected number as first argument")
if !isNumber(call.Argument(0)) {
return nil, fmt.Errorf("expected number as first argument")
}
blocks = call.Argument(0).ToInteger()
}
if nArgs >= 2 {
if call.Argument(1).IsNumber() {
sleep, _ = call.Argument(1).ToInteger()
} else {
throwJSException("expected number as second argument")
if isNumber(call.Argument(1)) {
return nil, fmt.Errorf("expected number as second argument")
}
sleep = call.Argument(1).ToInteger()
}
// go through the console, this will allow web3 to call the appropriate
// callbacks if a delayed response or notification is received.
blockNumber := func() int64 {
result, err := call.Otto.Run("eth.blockNumber")
if err != nil {
throwJSException(err.Error())
}
block, err := result.ToInteger()
if err != nil {
throwJSException(err.Error())
}
return block
}
// Poll the current block number until either it ot a timeout is reached
targetBlockNr := blockNumber() + blocks
deadline := time.Now().Add(time.Duration(sleep) * time.Second)
// Poll the current block number until either it or a timeout is reached.
var (
deadline = time.Now().Add(time.Duration(sleep) * time.Second)
lastNumber = ^hexutil.Uint64(0)
)
for time.Now().Before(deadline) {
if blockNumber() >= targetBlockNr {
return otto.TrueValue()
var number hexutil.Uint64
err := b.client.Call(&number, "eth_blockNumber")
if err != nil {
return nil, err
}
if number != lastNumber {
lastNumber = number
blocks--
}
if blocks <= 0 {
break
}
time.Sleep(time.Second)
}
return otto.FalseValue()
return call.VM.ToValue(true), nil
}
type jsonrpcCall struct {
@ -357,15 +370,15 @@ type jsonrpcCall struct {
}
// Send implements the web3 provider "send" method.
func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) {
func (b *bridge) Send(call jsre.Call) (goja.Value, error) {
// Remarshal the request into a Go value.
JSON, _ := call.Otto.Object("JSON")
reqVal, err := JSON.Call("stringify", call.Argument(0))
reqVal, err := call.Argument(0).ToObject(call.VM).MarshalJSON()
if err != nil {
throwJSException(err.Error())
return nil, err
}
var (
rawReq = reqVal.String()
rawReq = string(reqVal)
dec = json.NewDecoder(strings.NewReader(rawReq))
reqs []jsonrpcCall
batch bool
@ -381,10 +394,12 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) {
}
// Execute the requests.
resps, _ := call.Otto.Object("new Array()")
var resps []*goja.Object
for _, req := range reqs {
resp, _ := call.Otto.Object(`({"jsonrpc":"2.0"})`)
resp := call.VM.NewObject()
resp.Set("jsonrpc", "2.0")
resp.Set("id", req.ID)
var result json.RawMessage
err = b.client.Call(&result, req.Method, req.Params...)
switch err := err.(type) {
@ -392,9 +407,14 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) {
if result == nil {
// Special case null because it is decoded as an empty
// raw message for some reason.
resp.Set("result", otto.NullValue())
resp.Set("result", goja.Null())
} else {
resultVal, err := JSON.Call("parse", string(result))
JSON := call.VM.Get("JSON").ToObject(call.VM)
parse, callable := goja.AssertFunction(JSON.Get("parse"))
if !callable {
return nil, fmt.Errorf("JSON.parse is not a function")
}
resultVal, err := parse(goja.Null(), call.VM.ToValue(string(result)))
if err != nil {
setError(resp, -32603, err.Error())
} else {
@ -406,33 +426,38 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) {
default:
setError(resp, -32603, err.Error())
}
resps.Call("push", resp)
resps = append(resps, resp)
}
// Return the responses either to the callback (if supplied)
// or directly as the return value.
var result goja.Value
if batch {
response = resps.Value()
result = call.VM.ToValue(resps)
} else {
response, _ = resps.Get("0")
result = resps[0]
}
if fn := call.Argument(1); fn.Class() == "Function" {
fn.Call(otto.NullValue(), otto.NullValue(), response)
return otto.UndefinedValue()
if fn, isFunc := goja.AssertFunction(call.Argument(1)); isFunc {
fn(goja.Null(), goja.Null(), result)
return goja.Undefined(), nil
}
return response
return result, nil
}
func setError(resp *otto.Object, code int, msg string) {
func setError(resp *goja.Object, code int, msg string) {
resp.Set("error", map[string]interface{}{"code": code, "message": msg})
}
// throwJSException panics on an otto.Value. The Otto VM will recover from the
// Go panic and throw msg as a JavaScript error.
func throwJSException(msg interface{}) otto.Value {
val, err := otto.ToValue(msg)
if err != nil {
log.Error("Failed to serialize JavaScript exception", "exception", msg, "err", err)
}
panic(val)
// isNumber returns true if input value is a JS number.
func isNumber(v goja.Value) bool {
k := v.ExportType().Kind()
return k >= reflect.Int && k <= reflect.Float64
}
func getObject(vm *goja.Runtime, name string) *goja.Object {
v := vm.Get(name)
if v == nil {
return nil
}
return v.ToObject(vm)
}

306
console/console.go
View file

@ -28,12 +28,13 @@ import (
"strings"
"syscall"
"github.com/dop251/goja"
"github.com/ethereum/go-ethereum/internal/jsre"
"github.com/ethereum/go-ethereum/internal/jsre/deps"
"github.com/ethereum/go-ethereum/internal/web3ext"
"github.com/ethereum/go-ethereum/rpc"
"github.com/mattn/go-colorable"
"github.com/peterh/liner"
"github.com/robertkrimen/otto"
)
var (
@ -86,6 +87,7 @@ func New(config Config) (*Console, error) {
if config.Printer == nil {
config.Printer = colorable.NewColorableStdout()
}
// Initialize the console and return
console := &Console{
client: config.Client,
@ -107,110 +109,35 @@ func New(config Config) (*Console, error) {
// init retrieves the available APIs from the remote RPC provider and initializes
// the console's JavaScript namespaces based on the exposed modules.
func (c *Console) init(preload []string) error {
// Initialize the JavaScript <-> Go RPC bridge
c.initConsoleObject()
// Initialize the JavaScript <-> Go RPC bridge.
bridge := newBridge(c.client, c.prompter, c.printer)
c.jsre.Set("jeth", struct{}{})
jethObj, _ := c.jsre.Get("jeth")
jethObj.Object().Set("send", bridge.Send)
jethObj.Object().Set("sendAsync", bridge.Send)
consoleObj, _ := c.jsre.Get("console")
consoleObj.Object().Set("log", c.consoleOutput)
consoleObj.Object().Set("error", c.consoleOutput)
// Load all the internal utility JavaScript libraries
if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil {
return fmt.Errorf("bignumber.js: %v", err)
}
if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil {
return fmt.Errorf("web3.js: %v", err)
}
if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
return fmt.Errorf("web3 require: %v", err)
}
if _, err := c.jsre.Run("var web3 = new Web3(jeth);"); err != nil {
return fmt.Errorf("web3 provider: %v", err)
}
// Load the supported APIs into the JavaScript runtime environment
apis, err := c.client.SupportedModules()
if err != nil {
return fmt.Errorf("api modules: %v", err)
}
flatten := "var eth = web3.eth; var personal = web3.personal; "
for api := range apis {
if api == "web3" {
continue // manually mapped or ignore
}
if file, ok := web3ext.Modules[api]; ok {
// Load our extension for the module.
if err = c.jsre.Compile(fmt.Sprintf("%s.js", api), file); err != nil {
return fmt.Errorf("%s.js: %v", api, err)
}
flatten += fmt.Sprintf("var %s = web3.%s; ", api, api)
} else if obj, err := c.jsre.Run("web3." + api); err == nil && obj.IsObject() {
// Enable web3.js built-in extension if available.
flatten += fmt.Sprintf("var %s = web3.%s; ", api, api)
}
}
if _, err = c.jsre.Run(flatten); err != nil {
return fmt.Errorf("namespace flattening: %v", err)
}
// Initialize the global name register (disabled for now)
//c.jsre.Run(`var GlobalRegistrar = eth.contract(` + registrar.GlobalRegistrarAbi + `); registrar = GlobalRegistrar.at("` + registrar.GlobalRegistrarAddr + `");`)
// If the console is in interactive mode, instrument password related methods to query the user
if c.prompter != nil {
// Retrieve the account management object to instrument
personal, err := c.jsre.Get("personal")
if err != nil {
return err
}
// Override the openWallet, unlockAccount, newAccount and sign methods since
// these require user interaction. Assign these method in the Console the
// original web3 callbacks. These will be called by the jeth.* methods after
// they got the password from the user and send the original web3 request to
// the backend.
if obj := personal.Object(); obj != nil { // make sure the personal api is enabled over the interface
if _, err = c.jsre.Run(`jeth.openWallet = personal.openWallet;`); err != nil {
return fmt.Errorf("personal.openWallet: %v", err)
}
if _, err = c.jsre.Run(`jeth.unlockAccount = personal.unlockAccount;`); err != nil {
return fmt.Errorf("personal.unlockAccount: %v", err)
}
if _, err = c.jsre.Run(`jeth.newAccount = personal.newAccount;`); err != nil {
return fmt.Errorf("personal.newAccount: %v", err)
}
if _, err = c.jsre.Run(`jeth.sign = personal.sign;`); err != nil {
return fmt.Errorf("personal.sign: %v", err)
}
obj.Set("openWallet", bridge.OpenWallet)
obj.Set("unlockAccount", bridge.UnlockAccount)
obj.Set("newAccount", bridge.NewAccount)
obj.Set("sign", bridge.Sign)
}
}
// The admin.sleep and admin.sleepBlocks are offered by the console and not by the RPC layer.
admin, err := c.jsre.Get("admin")
if err != nil {
if err := c.initWeb3(bridge); err != nil {
return err
}
if obj := admin.Object(); obj != nil { // make sure the admin api is enabled over the interface
obj.Set("sleepBlocks", bridge.SleepBlocks)
obj.Set("sleep", bridge.Sleep)
obj.Set("clearHistory", c.clearHistory)
if err := c.initExtensions(); err != nil {
return err
}
// Preload any JavaScript files before starting the console
// Add bridge overrides for web3.js functionality.
c.jsre.Do(func(vm *goja.Runtime) {
c.initAdmin(vm, bridge)
c.initPersonal(vm, bridge)
})
// Preload JavaScript files.
for _, path := range preload {
if err := c.jsre.Exec(path); err != nil {
failure := err.Error()
if ottoErr, ok := err.(*otto.Error); ok {
failure = ottoErr.String()
if gojaErr, ok := err.(*goja.Exception); ok {
failure = gojaErr.String()
}
return fmt.Errorf("%s: %v", path, failure)
}
}
// Configure the console's input prompter for scrollback and tab completion
// Configure the input prompter for history and tab completion.
if c.prompter != nil {
if content, err := ioutil.ReadFile(c.histPath); err != nil {
c.prompter.SetHistory(nil)
@ -223,6 +150,102 @@ func (c *Console) init(preload []string) error {
return nil
}
func (c *Console) initConsoleObject() {
c.jsre.Do(func(vm *goja.Runtime) {
console := vm.NewObject()
console.Set("log", c.consoleOutput)
console.Set("error", c.consoleOutput)
vm.Set("console", console)
})
}
func (c *Console) initWeb3(bridge *bridge) error {
bnJS := string(deps.MustAsset("bignumber.js"))
web3JS := string(deps.MustAsset("web3.js"))
if err := c.jsre.Compile("bignumber.js", bnJS); err != nil {
return fmt.Errorf("bignumber.js: %v", err)
}
if err := c.jsre.Compile("web3.js", web3JS); err != nil {
return fmt.Errorf("web3.js: %v", err)
}
if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
return fmt.Errorf("web3 require: %v", err)
}
var err error
c.jsre.Do(func(vm *goja.Runtime) {
transport := vm.NewObject()
transport.Set("send", jsre.MakeCallback(vm, bridge.Send))
transport.Set("sendAsync", jsre.MakeCallback(vm, bridge.Send))
vm.Set("_consoleWeb3Transport", transport)
_, err = vm.RunString("var web3 = new Web3(_consoleWeb3Transport)")
})
return err
}
// initExtensions loads and registers web3.js extensions.
func (c *Console) initExtensions() error {
// Compute aliases from server-provided modules.
apis, err := c.client.SupportedModules()
if err != nil {
return fmt.Errorf("api modules: %v", err)
}
aliases := map[string]struct{}{"eth": {}, "personal": {}}
for api := range apis {
if api == "web3" {
continue
}
aliases[api] = struct{}{}
if file, ok := web3ext.Modules[api]; ok {
if err = c.jsre.Compile(api+".js", file); err != nil {
return fmt.Errorf("%s.js: %v", api, err)
}
}
}
// Apply aliases.
c.jsre.Do(func(vm *goja.Runtime) {
web3 := getObject(vm, "web3")
for name := range aliases {
if v := web3.Get(name); v != nil {
vm.Set(name, v)
}
}
})
return nil
}
// initAdmin creates additional admin APIs implemented by the bridge.
func (c *Console) initAdmin(vm *goja.Runtime, bridge *bridge) {
if admin := getObject(vm, "admin"); admin != nil {
admin.Set("sleepBlocks", jsre.MakeCallback(vm, bridge.SleepBlocks))
admin.Set("sleep", jsre.MakeCallback(vm, bridge.Sleep))
admin.Set("clearHistory", c.clearHistory)
}
}
// initPersonal redirects account-related API methods through the bridge.
//
// If the console is in interactive mode and the 'personal' API is available, override
// the openWallet, unlockAccount, newAccount and sign methods since these require user
// interaction. The original web3 callbacks are stored in 'jeth'. These will be called
// by the bridge after the prompt and send the original web3 request to the backend.
func (c *Console) initPersonal(vm *goja.Runtime, bridge *bridge) {
personal := getObject(vm, "personal")
if personal == nil || c.prompter == nil {
return
}
jeth := vm.NewObject()
vm.Set("jeth", jeth)
jeth.Set("openWallet", personal.Get("openWallet"))
jeth.Set("unlockAccount", personal.Get("unlockAccount"))
jeth.Set("newAccount", personal.Get("newAccount"))
jeth.Set("sign", personal.Get("sign"))
personal.Set("openWallet", jsre.MakeCallback(vm, bridge.OpenWallet))
personal.Set("unlockAccount", jsre.MakeCallback(vm, bridge.UnlockAccount))
personal.Set("newAccount", jsre.MakeCallback(vm, bridge.NewAccount))
personal.Set("sign", jsre.MakeCallback(vm, bridge.Sign))
}
func (c *Console) clearHistory() {
c.history = nil
c.prompter.ClearHistory()
@ -235,13 +258,13 @@ func (c *Console) clearHistory() {
// consoleOutput is an override for the console.log and console.error methods to
// stream the output into the configured output stream instead of stdout.
func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {
func (c *Console) consoleOutput(call goja.FunctionCall) goja.Value {
var output []string
for _, argument := range call.ArgumentList {
for _, argument := range call.Arguments {
output = append(output, fmt.Sprintf("%v", argument))
}
fmt.Fprintln(c.printer, strings.Join(output, " "))
return otto.Value{}
return goja.Null()
}
// AutoCompleteInput is a pre-assembled word completer to be used by the user
@ -304,75 +327,74 @@ func (c *Console) Welcome() {
// Evaluate executes code and pretty prints the result to the specified output
// stream.
func (c *Console) Evaluate(statement string) error {
func (c *Console) Evaluate(statement string) {
defer func() {
if r := recover(); r != nil {
fmt.Fprintf(c.printer, "[native] error: %v\n", r)
}
}()
return c.jsre.Evaluate(statement, c.printer)
c.jsre.Evaluate(statement, c.printer)
}
// Interactive starts an interactive user session, where input is propted from
// the configured user prompter.
func (c *Console) Interactive() {
var (
prompt = c.prompt // Current prompt line (used for multi-line inputs)
indents = 0 // Current number of input indents (used for multi-line inputs)
input = "" // Current user input
scheduler = make(chan string) // Channel to send the next prompt on and receive the input
prompt = c.prompt // the current prompt line (used for multi-line inputs)
indents = 0 // the current number of input indents (used for multi-line inputs)
input = "" // the current user input
inputLine = make(chan string, 1) // receives user input
inputErr = make(chan error, 1) // receives liner errors
requestLine = make(chan string) // requests a line of input
interrupt = make(chan os.Signal, 1)
)
// Start a goroutine to listen for prompt requests and send back inputs
go func() {
for {
// Read the next user input
line, err := c.prompter.PromptInput(<-scheduler)
if err != nil {
// In case of an error, either clear the prompt or fail
if err == liner.ErrPromptAborted { // ctrl-C
prompt, indents, input = c.prompt, 0, ""
scheduler <- ""
continue
}
close(scheduler)
return
}
// User input retrieved, send for interpretation and loop
scheduler <- line
}
}()
// Monitor Ctrl-C too in case the input is empty and we need to bail
abort := make(chan os.Signal, 1)
signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
// Start sending prompts to the user and reading back inputs
// Monitor Ctrl-C. While liner does turn on the relevant terminal mode bits to avoid
// the signal, a signal can still be received for unsupported terminals. Unfortunately
// there is no way to cancel the line reader when this happens. The readLines
// goroutine will be leaked in this case.
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(interrupt)
// The line reader runs in a separate goroutine.
go c.readLines(inputLine, inputErr, requestLine)
defer close(requestLine)
for {
// Send the next prompt, triggering an input read and process the result
scheduler <- prompt
// Send the next prompt, triggering an input read.
requestLine <- prompt
select {
case <-abort:
// User forcefully quite the console
case <-interrupt:
fmt.Fprintln(c.printer, "caught interrupt, exiting")
return
case line, ok := <-scheduler:
// User input was returned by the prompter, handle special cases
if !ok || (indents <= 0 && exit.MatchString(line)) {
case err := <-inputErr:
if err == liner.ErrPromptAborted && indents > 0 {
// When prompting for multi-line input, the first Ctrl-C resets
// the multi-line state.
prompt, indents, input = c.prompt, 0, ""
continue
}
return
case line := <-inputLine:
// User input was returned by the prompter, handle special cases.
if indents <= 0 && exit.MatchString(line) {
return
}
if onlyWhitespace.MatchString(line) {
continue
}
// Append the line to the input and check for multi-line interpretation
// Append the line to the input and check for multi-line interpretation.
input += line + "\n"
indents = countIndents(input)
if indents <= 0 {
prompt = c.prompt
} else {
prompt = strings.Repeat(".", indents*3) + " "
}
// If all the needed lines are present, save the command and run
// If all the needed lines are present, save the command and run it.
if indents <= 0 {
if len(input) > 0 && input[0] != ' ' && !passwordRegexp.MatchString(input) {
if command := strings.TrimSpace(input); len(c.history) == 0 || command != c.history[len(c.history)-1] {
@ -389,6 +411,18 @@ func (c *Console) Interactive() {
}
}
// readLines runs in its own goroutine, prompting for input.
func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-chan string) {
for p := range prompt {
line, err := c.prompter.PromptInput(p)
if err != nil {
errc <- err
} else {
input <- line
}
}
}
// countIndents returns the number of identations for the given input.
// In case of invalid input such as var a = } the result can be negative.
func countIndents(input string) int {

2
console/console_test.go
View file

@ -289,7 +289,7 @@ func TestPrettyError(t *testing.T) {
defer tester.Close(t)
tester.console.Evaluate("throw 'hello'")
want := jsre.ErrorColor("hello") + "\n"
want := jsre.ErrorColor("hello") + "\n\tat <eval>:1:7(1)\n\n"
if output := tester.output.String(); output != want {
t.Fatalf("pretty error mismatch: have %s, want %s", output, want)
}

227
core/blockchain.go
View file

@ -34,6 +34,7 @@ import (
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb"
@ -61,6 +62,10 @@ var (
storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil)
storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil)
snapshotAccountReadTimer = metrics.NewRegisteredTimer("chain/snapshot/account/reads", nil)
snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil)
snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil)
blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil)
@ -115,6 +120,9 @@ type CacheConfig struct {
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node)
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
}
// BlockChain represents the canonical chain given a database with a genesis
@ -136,6 +144,7 @@ type BlockChain struct {
cacheConfig *CacheConfig // Cache configuration for pruning
db ethdb.Database // Low level persistent database to store final content in
snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping
@ -188,6 +197,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: true,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
@ -293,6 +304,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
}
}
}
// Load any existing snapshot, regenerating it if loading failed
if bc.cacheConfig.SnapshotLimit > 0 {
bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, bc.CurrentBlock().Root(), !bc.cacheConfig.SnapshotWait)
}
// Take ownership of this particular state
go bc.update()
return bc, nil
@ -339,7 +354,7 @@ func (bc *BlockChain) loadLastState() error {
return bc.Reset()
}
// Make sure the state associated with the block is available
if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
if _, err := state.New(currentBlock.Root(), bc.stateCache, bc.snaps); err != nil {
// Dangling block without a state associated, init from scratch
log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
if err := bc.repair(&currentBlock); err != nil {
@ -401,12 +416,17 @@ func (bc *BlockChain) SetHead(head uint64) error {
if newHeadBlock == nil {
newHeadBlock = bc.genesisBlock
} else {
if _, err := state.New(newHeadBlock.Root(), bc.stateCache); err != nil {
if _, err := state.New(newHeadBlock.Root(), bc.stateCache, bc.snaps); err != nil {
// Rewound state missing, rolled back to before pivot, reset to genesis
newHeadBlock = bc.genesisBlock
}
}
rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash())
// Degrade the chain markers if they are explicitly reverted.
// In theory we should update all in-memory markers in the
// last step, however the direction of SetHead is from high
// to low, so it's safe the update in-memory markers directly.
bc.currentBlock.Store(newHeadBlock)
headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
}
@ -419,6 +439,11 @@ func (bc *BlockChain) SetHead(head uint64) error {
newHeadFastBlock = bc.genesisBlock
}
rawdb.WriteHeadFastBlockHash(db, newHeadFastBlock.Hash())
// Degrade the chain markers if they are explicitly reverted.
// In theory we should update all in-memory markers in the
// last step, however the direction of SetHead is from high
// to low, so it's safe the update in-memory markers directly.
bc.currentFastBlock.Store(newHeadFastBlock)
headFastBlockGauge.Update(int64(newHeadFastBlock.NumberU64()))
}
@ -476,6 +501,10 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
headBlockGauge.Update(int64(block.NumberU64()))
bc.chainmu.Unlock()
// Destroy any existing state snapshot and regenerate it in the background
if bc.snaps != nil {
bc.snaps.Rebuild(block.Root())
}
log.Info("Committed new head block", "number", block.Number(), "hash", hash)
return nil
}
@ -491,6 +520,15 @@ func (bc *BlockChain) CurrentBlock() *types.Block {
return bc.currentBlock.Load().(*types.Block)
}
// Snapshot returns the blockchain snapshot tree. This method is mainly used for
// testing, to make it possible to verify the snapshot after execution.
//
// Warning: There are no guarantees about the safety of using the returned 'snap' if the
// blockchain is simultaneously importing blocks, so take care.
func (bc *BlockChain) Snapshot() *snapshot.Tree {
return bc.snaps
}
// CurrentFastBlock retrieves the current fast-sync head block of the canonical
// chain. The block is retrieved from the blockchain's internal cache.
func (bc *BlockChain) CurrentFastBlock() *types.Block {
@ -514,7 +552,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) {
// StateAt returns a new mutable state based on a particular point in time.
func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.stateCache)
return state.New(root, bc.stateCache, bc.snaps)
}
// StateCache returns the caching database underpinning the blockchain instance.
@ -538,21 +576,22 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
defer bc.chainmu.Unlock()
// Prepare the genesis block and reinitialise the chain
if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil {
log.Crit("Failed to write genesis block TD", "err", err)
batch := bc.db.NewBatch()
rawdb.WriteTd(batch, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty())
rawdb.WriteBlock(batch, genesis)
if err := batch.Write(); err != nil {
log.Crit("Failed to write genesis block", "err", err)
}
rawdb.WriteBlock(bc.db, genesis)
bc.writeHeadBlock(genesis)
// Last update all in-memory chain markers
bc.genesisBlock = genesis
bc.insert(bc.genesisBlock)
bc.currentBlock.Store(bc.genesisBlock)
headBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
bc.hc.SetGenesis(bc.genesisBlock.Header())
bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
bc.currentFastBlock.Store(bc.genesisBlock)
headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
return nil
}
@ -565,7 +604,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
func (bc *BlockChain) repair(head **types.Block) error {
for {
// Abort if we've rewound to a head block that does have associated state
if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
if _, err := state.New((*head).Root(), bc.stateCache, bc.snaps); err == nil {
log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash())
return nil
}
@ -610,31 +649,39 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
return nil
}
// insert injects a new head block into the current block chain. This method
// writeHeadBlock injects a new head block into the current block chain. This method
// assumes that the block is indeed a true head. It will also reset the head
// header and the head fast sync block to this very same block if they are older
// or if they are on a different side chain.
//
// Note, this function assumes that the `mu` mutex is held!
func (bc *BlockChain) insert(block *types.Block) {
func (bc *BlockChain) writeHeadBlock(block *types.Block) {
// If the block is on a side chain or an unknown one, force other heads onto it too
updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash()
// Add the block to the canonical chain number scheme and mark as the head
rawdb.WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64())
rawdb.WriteHeadBlockHash(bc.db, block.Hash())
bc.currentBlock.Store(block)
headBlockGauge.Update(int64(block.NumberU64()))
batch := bc.db.NewBatch()
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
rawdb.WriteTxLookupEntries(batch, block)
rawdb.WriteHeadBlockHash(batch, block.Hash())
// If the block is better than our head or is on a different chain, force update heads
if updateHeads {
rawdb.WriteHeadHeaderHash(batch, block.Hash())
rawdb.WriteHeadFastBlockHash(batch, block.Hash())
}
// Flush the whole batch into the disk, exit the node if failed
if err := batch.Write(); err != nil {
log.Crit("Failed to update chain indexes and markers", "err", err)
}
// Update all in-memory chain markers in the last step
if updateHeads {
bc.hc.SetCurrentHeader(block.Header())
rawdb.WriteHeadFastBlockHash(bc.db, block.Hash())
bc.currentFastBlock.Store(block)
headFastBlockGauge.Update(int64(block.NumberU64()))
}
bc.currentBlock.Store(block)
headBlockGauge.Update(int64(block.NumberU64()))
}
// Genesis retrieves the chain's genesis block.
@ -820,6 +867,14 @@ func (bc *BlockChain) Stop() {
bc.wg.Wait()
// Ensure that the entirety of the state snapshot is journalled to disk.
var snapBase common.Hash
if bc.snaps != nil {
var err error
if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil {
log.Error("Failed to journal state snapshot", "err", err)
}
}
// Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios:
// - HEAD: So we don't need to reprocess any blocks in the general case
@ -838,6 +893,12 @@ func (bc *BlockChain) Stop() {
}
}
}
if snapBase != (common.Hash{}) {
log.Info("Writing snapshot state to disk", "root", snapBase)
if err := triedb.Commit(snapBase, true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
}
}
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem().(common.Hash))
}
@ -845,7 +906,7 @@ func (bc *BlockChain) Stop() {
log.Error("Dangling trie nodes after full cleanup")
}
}
log.Info("Blockchain manager stopped")
log.Info("Blockchain stopped")
}
func (bc *BlockChain) procFutureBlocks() {
@ -881,26 +942,36 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
bc.chainmu.Lock()
defer bc.chainmu.Unlock()
batch := bc.db.NewBatch()
for i := len(chain) - 1; i >= 0; i-- {
hash := chain[i]
// Degrade the chain markers if they are explicitly reverted.
// In theory we should update all in-memory markers in the
// last step, however the direction of rollback is from high
// to low, so it's safe the update in-memory markers directly.
currentHeader := bc.hc.CurrentHeader()
if currentHeader.Hash() == hash {
bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1))
newHeadHeader := bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
rawdb.WriteHeadHeaderHash(batch, currentHeader.ParentHash)
bc.hc.SetCurrentHeader(newHeadHeader)
}
if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash {
newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1)
rawdb.WriteHeadFastBlockHash(bc.db, newFastBlock.Hash())
rawdb.WriteHeadFastBlockHash(batch, currentFastBlock.ParentHash())
bc.currentFastBlock.Store(newFastBlock)
headFastBlockGauge.Update(int64(newFastBlock.NumberU64()))
}
if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash {
newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1)
rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash())
rawdb.WriteHeadBlockHash(batch, currentBlock.ParentHash())
bc.currentBlock.Store(newBlock)
headBlockGauge.Update(int64(newBlock.NumberU64()))
}
}
if err := batch.Write(); err != nil {
log.Crit("Failed to rollback chain markers", "err", err)
}
// Truncate ancient data which exceeds the current header.
//
// Notably, it can happen that system crashes without truncating the ancient data
@ -1063,7 +1134,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
}
// Don't collect too much in-memory, write it out every 100K blocks
if len(deleted) > 100000 {
// Sync the ancient store explicitly to ensure all data has been flushed to disk.
if err := bc.db.Sync(); err != nil {
return 0, err
@ -1172,7 +1242,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
rawdb.WriteTxLookupEntries(batch, block)
stats.processed++
// Write everything belongs to the blocks into the database. So that
// we can ensure all components of body is completed(body, receipts,
// tx indexes)
if batch.ValueSize() >= ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return 0, err
@ -1180,7 +1252,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
size += batch.ValueSize()
batch.Reset()
}
stats.processed++
}
// Write everything belongs to the blocks into the database. So that
// we can ensure all components of body is completed(body, receipts,
// tx indexes)
if batch.ValueSize() > 0 {
size += batch.ValueSize()
if err := batch.Write(); err != nil {
@ -1231,11 +1307,12 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e
bc.wg.Add(1)
defer bc.wg.Done()
if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil {
return err
batch := bc.db.NewBatch()
rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td)
rawdb.WriteBlock(batch, block)
if err := batch.Write(); err != nil {
log.Crit("Failed to write block into disk", "err", err)
}
rawdb.WriteBlock(bc.db, block)
return nil
}
@ -1251,11 +1328,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error {
return err
}
}
// Write the positional metadata for transaction/receipt lookups.
// Preimages here is empty, ignore it.
rawdb.WriteTxLookupEntries(bc.db, block)
bc.insert(block)
bc.writeHeadBlock(block)
return nil
}
@ -1283,12 +1356,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
externTd := new(big.Int).Add(block.Difficulty(), ptd)
// Irrelevant of the canonical status, write the block itself to the database
if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil {
return NonStatTy, err
// Irrelevant of the canonical status, write the block itself to the database.
//
// Note all the components of block(td, hash->number map, header, body, receipts)
// should be written atomically. BlockBatch is used for containing all components.
blockBatch := bc.db.NewBatch()
rawdb.WriteTd(blockBatch, block.Hash(), block.NumberU64(), externTd)
rawdb.WriteBlock(blockBatch, block)
rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts)
rawdb.WritePreimages(blockBatch, state.Preimages())
if err := blockBatch.Write(); err != nil {
log.Crit("Failed to write block into disk", "err", err)
}
rawdb.WriteBlock(bc.db, block)
// Commit all cached state changes into underlying memory database.
root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number()))
if err != nil {
return NonStatTy, err
@ -1347,11 +1427,6 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
}
}
}
// Write other block data using a batch.
batch := bc.db.NewBatch()
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts)
// If the total difficulty is higher than our known, add it to the canonical chain
// Second clause in the if statement reduces the vulnerability to selfish mining.
// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
@ -1377,21 +1452,13 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
return NonStatTy, err
}
}
// Write the positional metadata for transaction/receipt lookups and preimages
rawdb.WriteTxLookupEntries(batch, block)
rawdb.WritePreimages(batch, state.Preimages())
status = CanonStatTy
} else {
status = SideStatTy
}
if err := batch.Write(); err != nil {
return NonStatTy, err
}
// Set new head.
if status == CanonStatTy {
bc.insert(block)
bc.writeHeadBlock(block)
}
bc.futureBlocks.Remove(block.Hash())
@ -1622,25 +1689,24 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
if parent == nil {
parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
}
statedb, err := state.New(parent.Root, bc.stateCache)
statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
if err != nil {
return it.index, err
}
// If we have a followup block, run that against the current state to pre-cache
// transactions and probabilistically some of the account/storage trie nodes.
var followupInterrupt uint32
if !bc.cacheConfig.TrieCleanNoPrefetch {
if followup, err := it.peek(); followup != nil && err == nil {
go func(start time.Time) {
throwaway, _ := state.New(parent.Root, bc.stateCache)
throwaway, _ := state.New(parent.Root, bc.stateCache, bc.snaps)
go func(start time.Time, followup *types.Block, throwaway *state.StateDB, interrupt *uint32) {
bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, &followupInterrupt)
blockPrefetchExecuteTimer.Update(time.Since(start))
if atomic.LoadUint32(&followupInterrupt) == 1 {
if atomic.LoadUint32(interrupt) == 1 {
blockPrefetchInterruptMeter.Mark(1)
}
}(time.Now())
}(time.Now(), followup, throwaway, &followupInterrupt)
}
}
// Process block using the parent state as reference point
@ -1652,14 +1718,16 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
return it.index, err
}
// Update the metrics touched during block processing
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them
storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them
accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them
storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them
storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them
accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them
storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them
snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them
snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them
triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation
trieproc := statedb.AccountReads + statedb.AccountUpdates
trieproc += statedb.StorageReads + statedb.StorageUpdates
trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates
trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates
blockExecutionTimer.Update(time.Since(substart) - trieproc - triehash)
@ -1681,17 +1749,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
// Write the block to the chain and get the status.
substart = time.Now()
status, err := bc.writeBlockWithState(block, receipts, logs, statedb, false)
atomic.StoreUint32(&followupInterrupt, 1)
if err != nil {
atomic.StoreUint32(&followupInterrupt, 1)
return it.index, err
}
atomic.StoreUint32(&followupInterrupt, 1)
// Update the metrics touched during block commit
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits)
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits)
blockInsertTimer.UpdateSince(start)
switch status {
@ -1991,20 +2059,19 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
// taking care of the proper incremental order.
for i := len(newChain) - 1; i >= 1; i-- {
// Insert the block in the canonical way, re-writing history
bc.insert(newChain[i])
bc.writeHeadBlock(newChain[i])
// Collect reborn logs due to chain reorg
collectLogs(newChain[i].Hash(), false)
// Write lookup entries for hash based transaction/receipt searches
rawdb.WriteTxLookupEntries(bc.db, newChain[i])
// Collect the new added transactions.
addedTxs = append(addedTxs, newChain[i].Transactions()...)
}
// When transactions get deleted from the database, the receipts that were
// created in the fork must also be deleted
batch := bc.db.NewBatch()
// Delete useless indexes right now which includes the non-canonical
// transaction indexes, canonical chain indexes which above the head.
indexesBatch := bc.db.NewBatch()
for _, tx := range types.TxDifference(deletedTxs, addedTxs) {
rawdb.DeleteTxLookupEntry(batch, tx.Hash())
rawdb.DeleteTxLookupEntry(indexesBatch, tx.Hash())
}
// Delete any canonical number assignments above the new head
number := bc.CurrentBlock().NumberU64()
@ -2013,9 +2080,11 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
if hash == (common.Hash{}) {
break
}
rawdb.DeleteCanonicalHash(batch, i)
rawdb.DeleteCanonicalHash(indexesBatch, i)
}
if err := indexesBatch.Write(); err != nil {
log.Crit("Failed to delete useless indexes", "err", err)
}
batch.Write()
// If any logs need to be fired, do it now. In theory we could avoid creating
// this goroutine if there are no events to fire, but realistcally that only
// ever happens if we're reorging empty blocks, which will only happen on idle

524
core/blockchain_test.go
View file

@ -144,7 +144,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
}
return err
}
statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache)
statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache, nil)
if err != nil {
return err
}
@ -973,6 +973,7 @@ func TestLogReorgs(t *testing.T) {
t.Fatalf("failed to insert forked chain: %v", err)
}
timeout := time.NewTimer(1 * time.Second)
defer timeout.Stop()
select {
case <-done:
case <-timeout.C:
@ -2315,7 +2316,7 @@ func TestDeleteCreateRevert(t *testing.T) {
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: []byte{byte(vm.PC), 0xFF},
Code: []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)},
Nonce: 1,
Balance: big.NewInt(0),
},
@ -2362,3 +2363,522 @@ func TestDeleteCreateRevert(t *testing.T) {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
}
// TestDeleteRecreateSlots tests a state-transition that contains both deletion
// and recreation of contract state.
// Contract A exists, has slots 1 and 2 set
// Tx 1: Selfdestruct A
// Tx 2: Re-create A, set slots 3 and 4
// Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct,
// and then the new slots exist
func TestDeleteRecreateSlots(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. Set slots 3=3, 4=4,
// 2. Return aaCode
initCode := []byte{
byte(vm.PUSH1), 0x3, // value
byte(vm.PUSH1), 0x3, // location
byte(vm.SSTORE), // Set slot[3] = 1
byte(vm.PUSH1), 0x4, // value
byte(vm.PUSH1), 0x4, // location
byte(vm.SSTORE), // Set slot[4] = 1
// Slots are set, now return the code
byte(vm.PUSH2), byte(vm.PC), byte(vm.SELFDESTRUCT), // Push code on stack
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
// Code is now in memory.
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), byte(32 - 2), // offset
byte(vm.RETURN),
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
// The contract BB recreates AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to AA, to kill it
tx, _ := types.SignTx(types.NewTransaction(0, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
// One transaction to BB, to recreate AA
tx, _ = types.SignTx(types.NewTransaction(1, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
Debug: true,
Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then slot 1 and 2 are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
// Also, 3 and 4 should be set
if got, exp := statedb.GetState(aa, common.HexToHash("03")), common.HexToHash("03"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("04")), common.HexToHash("04"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
}
}
// TestDeleteRecreateAccount tests a state-transition that contains deletion of a
// contract with storage, and a recreate of the same contract via a
// regular value-transfer
// Expected outcome is that _all_ slots are cleared from A
func TestDeleteRecreateAccount(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
},
}
genesis := gspec.MustCommit(db)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to AA, to kill it
tx, _ := types.SignTx(types.NewTransaction(0, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
// One transaction to AA, to recreate it (but without storage
tx, _ = types.SignTx(types.NewTransaction(1, aa,
big.NewInt(1), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
Debug: true,
Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then both slots are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
}
// TestDeleteRecreateSlotsAcrossManyBlocks tests multiple state-transition that contains both deletion
// and recreation of contract state.
// Contract A exists, has slots 1 and 2 set
// Tx 1: Selfdestruct A
// Tx 2: Re-create A, set slots 3 and 4
// Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct,
// and then the new slots exist
func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. Set slots 3=blocknum+1, 4=4,
// 2. Return aaCode
initCode := []byte{
byte(vm.PUSH1), 0x1, //
byte(vm.NUMBER), // value = number + 1
byte(vm.ADD), //
byte(vm.PUSH1), 0x3, // location
byte(vm.SSTORE), // Set slot[3] = number + 1
byte(vm.PUSH1), 0x4, // value
byte(vm.PUSH1), 0x4, // location
byte(vm.SSTORE), // Set slot[4] = 4
// Slots are set, now return the code
byte(vm.PUSH2), byte(vm.PC), byte(vm.SELFDESTRUCT), // Push code on stack
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
// Code is now in memory.
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), byte(32 - 2), // offset
byte(vm.RETURN),
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
// The contract BB recreates AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
var nonce uint64
type expectation struct {
exist bool
blocknum int
values map[int]int
}
var current = &expectation{
exist: true, // exists in genesis
blocknum: 0,
values: map[int]int{1: 1, 2: 2},
}
var expectations []*expectation
var newDestruct = func(e *expectation) *types.Transaction {
tx, _ := types.SignTx(types.NewTransaction(nonce, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
nonce++
if e.exist {
e.exist = false
e.values = nil
}
t.Logf("block %d; adding destruct\n", e.blocknum)
return tx
}
var newResurrect = func(e *expectation) *types.Transaction {
tx, _ := types.SignTx(types.NewTransaction(nonce, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
nonce++
if !e.exist {
e.exist = true
e.values = map[int]int{3: e.blocknum + 1, 4: 4}
}
t.Logf("block %d; adding resurrect\n", e.blocknum)
return tx
}
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 150, func(i int, b *BlockGen) {
var exp = new(expectation)
exp.blocknum = i + 1
exp.values = make(map[int]int)
for k, v := range current.values {
exp.values[k] = v
}
exp.exist = current.exist
b.SetCoinbase(common.Address{1})
if i%2 == 0 {
b.AddTx(newDestruct(exp))
}
if i%3 == 0 {
b.AddTx(newResurrect(exp))
}
if i%5 == 0 {
b.AddTx(newDestruct(exp))
}
if i%7 == 0 {
b.AddTx(newResurrect(exp))
}
expectations = append(expectations, exp)
current = exp
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
//Debug: true,
//Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
var asHash = func(num int) common.Hash {
return common.BytesToHash([]byte{byte(num)})
}
for i, block := range blocks {
blockNum := i + 1
if n, err := chain.InsertChain([]*types.Block{block}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then slot 1 and 2 are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
}
exp := expectations[i]
if exp.exist {
if !statedb.Exist(aa) {
t.Fatalf("block %d, expected %v to exist, it did not", blockNum, aa)
}
for slot, val := range exp.values {
if gotValue, expValue := statedb.GetState(aa, asHash(slot)), asHash(val); gotValue != expValue {
t.Fatalf("block %d, slot %d, got %x exp %x", blockNum, slot, gotValue, expValue)
}
}
} else {
if statedb.Exist(aa) {
t.Fatalf("block %d, expected %v to not exist, it did", blockNum, aa)
}
}
}
}
// TestInitThenFailCreateContract tests a pretty notorious case that happened
// on mainnet over blocks 7338108, 7338110 and 7338115.
// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated
// with 0.001 ether (thus created but no code)
// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on
// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the
// deployment fails due to OOG during initcode execution
// - Block 7338115: another tx checks the balance of
// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as
// zero.
//
// The problem being that the snapshotter maintains a destructset, and adds items
// to the destructset in case something is created "onto" an existing item.
// We need to either roll back the snapDestructs, or not place it into snapDestructs
// in the first place.
//
func TestInitThenFailCreateContract(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
)
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. If blocknum < 1, error out (e.g invalid opcode)
// 2. else, return a snippet of code
initCode := []byte{
byte(vm.PUSH1), 0x1, // y (2)
byte(vm.NUMBER), // x (number)
byte(vm.GT), // x > y?
byte(vm.PUSH1), byte(0x8),
byte(vm.JUMPI), // jump to label if number > 2
byte(0xFE), // illegal opcode
byte(vm.JUMPDEST),
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), 0x0, // offset
byte(vm.RETURN), // return 2 bytes of zero-code
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address aa has some funds
aa: {Balance: big.NewInt(100000)},
// The contract BB tries to create code onto AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
nonce := uint64(0)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 4, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to BB
tx, _ := types.SignTx(types.NewTransaction(nonce, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
nonce++
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
//Debug: true,
//Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
statedb, _ := chain.State()
if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 {
t.Fatalf("Genesis err, got %v exp %v", got, exp)
}
// First block tries to create, but fails
{
block := blocks[0]
if _, err := chain.InsertChain([]*types.Block{blocks[0]}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
}
statedb, _ = chain.State()
if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 {
t.Fatalf("block %d: got %v exp %v", block.NumberU64(), got, exp)
}
}
// Import the rest of the blocks
for _, block := range blocks[1:] {
if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
}
}
}

2
core/chain_indexer_test.go
View file

@ -203,7 +203,7 @@ func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, b
}
func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
firstChanged := headNum / b.indexer.sectionSize
firstChanged := (headNum + 1) / b.indexer.sectionSize
if firstChanged < b.stored {
b.stored = firstChanged
}

2
core/chain_makers.go
View file

@ -228,7 +228,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
return nil, nil
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(db))
statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil)
if err != nil {
panic(err)
}

36
core/evm.go
View file

@ -60,24 +60,32 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author
// GetHashFn returns a GetHashFunc which retrieves header hashes by number
func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash {
var cache map[uint64]common.Hash
// Cache will initially contain [refHash.parent],
// Then fill up with [refHash.p, refHash.pp, refHash.ppp, ...]
var cache []common.Hash
return func(n uint64) common.Hash {
// If there's no hash cache yet, make one
if cache == nil {
cache = map[uint64]common.Hash{
ref.Number.Uint64() - 1: ref.ParentHash,
if len(cache) == 0 {
cache = append(cache, ref.ParentHash)
}
if idx := ref.Number.Uint64() - n - 1; idx < uint64(len(cache)) {
return cache[idx]
}
// No luck in the cache, but we can start iterating from the last element we already know
lastKnownHash := cache[len(cache)-1]
lastKnownNumber := ref.Number.Uint64() - uint64(len(cache))
for {
header := chain.GetHeader(lastKnownHash, lastKnownNumber)
if header == nil {
break
}
}
// Try to fulfill the request from the cache
if hash, ok := cache[n]; ok {
return hash
}
// Not cached, iterate the blocks and cache the hashes
for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) {
cache[header.Number.Uint64()-1] = header.ParentHash
if n == header.Number.Uint64()-1 {
return header.ParentHash
cache = append(cache, header.ParentHash)
lastKnownHash = header.ParentHash
lastKnownNumber = header.Number.Uint64() - 1
if n == lastKnownNumber {
return lastKnownHash
}
}
return common.Hash{}

4
core/forkid/forkid_test.go
View file

@ -65,8 +65,8 @@ func TestCreation(t *testing.T) {
},
// Ropsten test cases
{
params.TestnetChainConfig,
params.TestnetGenesisHash,
params.RopstenChainConfig,
params.RopstenGenesisHash,
[]testcase{
{0, ID{Hash: checksumToBytes(0x30c7ddbc), Next: 10}}, // Unsynced, last Frontier, Homestead and first Tangerine block
{9, ID{Hash: checksumToBytes(0x30c7ddbc), Next: 10}}, // Last Tangerine block

30
core/genesis.go
View file

@ -152,10 +152,6 @@ func (e *GenesisMismatchError) Error() string {
//
// The returned chain configuration is never nil.
func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) {
return SetupGenesisBlockWithOverride(db, genesis, nil, nil)
}
func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideIstanbul, overrideMuirGlacier *big.Int) (*params.ChainConfig, common.Hash, error) {
if genesis != nil && genesis.Config == nil {
return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
}
@ -178,7 +174,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
// We have the genesis block in database(perhaps in ancient database)
// but the corresponding state is missing.
header := rawdb.ReadHeader(db, stored, 0)
if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0)); err != nil {
if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0), nil); err != nil {
if genesis == nil {
genesis = DefaultGenesisBlock()
}
@ -204,12 +200,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
// Get the existing chain configuration.
newcfg := genesis.configOrDefault(stored)
if overrideIstanbul != nil {
newcfg.IstanbulBlock = overrideIstanbul
}
if overrideMuirGlacier != nil {
newcfg.MuirGlacierBlock = overrideMuirGlacier
}
if err := newcfg.CheckConfigForkOrder(); err != nil {
return newcfg, common.Hash{}, err
}
@ -246,8 +236,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
return g.Config
case ghash == params.MainnetGenesisHash:
return params.MainnetChainConfig
case ghash == params.TestnetGenesisHash:
return params.TestnetChainConfig
case ghash == params.RopstenGenesisHash:
return params.RopstenChainConfig
case ghash == params.RinkebyGenesisHash:
return params.RinkebyChainConfig
case ghash == params.GoerliGenesisHash:
return params.GoerliChainConfig
default:
return params.AllEthashProtocolChanges
}
@ -259,7 +253,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
if db == nil {
db = rawdb.NewMemoryDatabase()
}
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil)
for addr, account := range g.Alloc {
statedb.AddBalance(addr, account.Balance)
statedb.SetCode(addr, account.Code)
@ -347,15 +341,15 @@ func DefaultGenesisBlock() *Genesis {
}
}
// DefaultTestnetGenesisBlock returns the Ropsten network genesis block.
func DefaultTestnetGenesisBlock() *Genesis {
// DefaultRopstenGenesisBlock returns the Ropsten network genesis block.
func DefaultRopstenGenesisBlock() *Genesis {
return &Genesis{
Config: params.TestnetChainConfig,
Config: params.RopstenChainConfig,
Nonce: 66,
ExtraData: hexutil.MustDecode("0x3535353535353535353535353535353535353535353535353535353535353535"),
GasLimit: 16777216,
Difficulty: big.NewInt(1048576),
Alloc: decodePrealloc(testnetAllocData),
Alloc: decodePrealloc(ropstenAllocData),
}
}

2
core/genesis_alloc.go
View file

File diff suppressed because one or more lines are too long

16
core/genesis_test.go
View file

@ -35,9 +35,9 @@ func TestDefaultGenesisBlock(t *testing.T) {
if block.Hash() != params.MainnetGenesisHash {
t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash)
}
block = DefaultTestnetGenesisBlock().ToBlock(nil)
if block.Hash() != params.TestnetGenesisHash {
t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash)
block = DefaultRopstenGenesisBlock().ToBlock(nil)
if block.Hash() != params.RopstenGenesisHash {
t.Errorf("wrong ropsten genesis hash, got %v, want %v", block.Hash(), params.RopstenGenesisHash)
}
}
@ -95,14 +95,14 @@ func TestSetupGenesis(t *testing.T) {
wantConfig: customg.Config,
},
{
name: "custom block in DB, genesis == testnet",
name: "custom block in DB, genesis == ropsten",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db)
return SetupGenesisBlock(db, DefaultTestnetGenesisBlock())
return SetupGenesisBlock(db, DefaultRopstenGenesisBlock())
},
wantErr: &GenesisMismatchError{Stored: customghash, New: params.TestnetGenesisHash},
wantHash: params.TestnetGenesisHash,
wantConfig: params.TestnetChainConfig,
wantErr: &GenesisMismatchError{Stored: customghash, New: params.RopstenGenesisHash},
wantHash: params.RopstenGenesisHash,
wantConfig: params.RopstenChainConfig,
},
{
name: "compatible config in DB",

85
core/headerchain.go
View file

@ -45,6 +45,14 @@ const (
// HeaderChain implements the basic block header chain logic that is shared by
// core.BlockChain and light.LightChain. It is not usable in itself, only as
// a part of either structure.
//
// HeaderChain is responsible for maintaining the header chain including the
// header query and updating.
//
// The components maintained by headerchain includes: (1) total difficult
// (2) header (3) block hash -> number mapping (4) canonical number -> hash mapping
// and (5) head header flag.
//
// It is not thread safe either, the encapsulating chain structures should do
// the necessary mutex locking/unlocking.
type HeaderChain struct {
@ -66,10 +74,8 @@ type HeaderChain struct {
engine consensus.Engine
}
// NewHeaderChain creates a new HeaderChain structure.
// getValidator should return the parent's validator
// procInterrupt points to the parent's interrupt semaphore
// wg points to the parent's shutdown wait group
// NewHeaderChain creates a new HeaderChain structure. ProcInterrupt points
// to the parent's interrupt semaphore.
func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, procInterrupt func() bool) (*HeaderChain, error) {
headerCache, _ := lru.New(headerCacheLimit)
tdCache, _ := lru.New(tdCacheLimit)
@ -147,25 +153,33 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
externTd := new(big.Int).Add(header.Difficulty, ptd)
// Irrelevant of the canonical status, write the td and header to the database
if err := hc.WriteTd(hash, number, externTd); err != nil {
log.Crit("Failed to write header total difficulty", "err", err)
//
// Note all the components of header(td, hash->number index and header) should
// be written atomically.
headerBatch := hc.chainDb.NewBatch()
rawdb.WriteTd(headerBatch, hash, number, externTd)
rawdb.WriteHeader(headerBatch, header)
if err := headerBatch.Write(); err != nil {
log.Crit("Failed to write header into disk", "err", err)
}
rawdb.WriteHeader(hc.chainDb, header)
// If the total difficulty is higher than our known, add it to the canonical chain
// Second clause in the if statement reduces the vulnerability to selfish mining.
// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) {
// If the header can be added into canonical chain, adjust the
// header chain markers(canonical indexes and head header flag).
//
// Note all markers should be written atomically.
// Delete any canonical number assignments above the new head
batch := hc.chainDb.NewBatch()
markerBatch := hc.chainDb.NewBatch()
for i := number + 1; ; i++ {
hash := rawdb.ReadCanonicalHash(hc.chainDb, i)
if hash == (common.Hash{}) {
break
}
rawdb.DeleteCanonicalHash(batch, i)
rawdb.DeleteCanonicalHash(markerBatch, i)
}
batch.Write()
// Overwrite any stale canonical number assignments
var (
@ -174,16 +188,19 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
headHeader = hc.GetHeader(headHash, headNumber)
)
for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash {
rawdb.WriteCanonicalHash(hc.chainDb, headHash, headNumber)
rawdb.WriteCanonicalHash(markerBatch, headHash, headNumber)
headHash = headHeader.ParentHash
headNumber = headHeader.Number.Uint64() - 1
headHeader = hc.GetHeader(headHash, headNumber)
}
// Extend the canonical chain with the new header
rawdb.WriteCanonicalHash(hc.chainDb, hash, number)
rawdb.WriteHeadHeaderHash(hc.chainDb, hash)
rawdb.WriteCanonicalHash(markerBatch, hash, number)
rawdb.WriteHeadHeaderHash(markerBatch, hash)
if err := markerBatch.Write(); err != nil {
log.Crit("Failed to write header markers into disk", "err", err)
}
// Last step update all in-memory head header markers
hc.currentHeaderHash = hash
hc.currentHeader.Store(types.CopyHeader(header))
headHeaderGauge.Update(header.Number.Int64())
@ -192,9 +209,9 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er
} else {
status = SideStatTy
}
hc.tdCache.Add(hash, externTd)
hc.headerCache.Add(hash, header)
hc.numberCache.Add(hash, number)
return
}
@ -396,14 +413,6 @@ func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int {
return hc.GetTd(hash, *number)
}
// WriteTd stores a block's total difficulty into the database, also caching it
// along the way.
func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error {
rawdb.WriteTd(hc.chainDb, hash, number, td)
hc.tdCache.Add(hash, new(big.Int).Set(td))
return nil
}
// GetHeader retrieves a block header from the database by hash and number,
// caching it if found.
func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header {
@ -431,6 +440,8 @@ func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header {
}
// HasHeader checks if a block header is present in the database or not.
// In theory, if header is present in the database, all relative components
// like td and hash->number should be present too.
func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool {
if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) {
return true
@ -458,10 +469,9 @@ func (hc *HeaderChain) CurrentHeader() *types.Header {
return hc.currentHeader.Load().(*types.Header)
}
// SetCurrentHeader sets the current head header of the canonical chain.
// SetCurrentHeader sets the in-memory head header marker of the canonical chan
// as the given header.
func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
rawdb.WriteHeadHeaderHash(hc.chainDb, head.Hash())
hc.currentHeader.Store(head)
hc.currentHeaderHash = head.Hash()
headHeaderGauge.Update(head.Number.Int64())
@ -500,11 +510,18 @@ func (hc *HeaderChain) SetHead(head uint64, updateFn UpdateHeadBlocksCallback, d
// first then remove the relative data from the database.
//
// Update head first(head fast block, head full block) before deleting the data.
markerBatch := hc.chainDb.NewBatch()
if updateFn != nil {
updateFn(hc.chainDb, parent)
updateFn(markerBatch, parent)
}
// Update head header then.
rawdb.WriteHeadHeaderHash(hc.chainDb, parentHash)
rawdb.WriteHeadHeaderHash(markerBatch, parentHash)
if err := markerBatch.Write(); err != nil {
log.Crit("Failed to update chain markers", "error", err)
}
hc.currentHeader.Store(parent)
hc.currentHeaderHash = parentHash
headHeaderGauge.Update(parent.Number.Int64())
// Remove the relative data from the database.
if delFn != nil {
@ -514,13 +531,11 @@ func (hc *HeaderChain) SetHead(head uint64, updateFn UpdateHeadBlocksCallback, d
rawdb.DeleteHeader(batch, hash, num)
rawdb.DeleteTd(batch, hash, num)
rawdb.DeleteCanonicalHash(batch, num)
hc.currentHeader.Store(parent)
hc.currentHeaderHash = parentHash
headHeaderGauge.Update(parent.Number.Int64())
}
batch.Write()
// Flush all accumulated deletions.
if err := batch.Write(); err != nil {
log.Crit("Failed to rewind block", "error", err)
}
// Clear out any stale content from the caches
hc.headerCache.Purge()
hc.tdCache.Purge()

2
core/rawdb/accessors_chain.go
View file

@ -69,7 +69,7 @@ func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash {
prefix := headerKeyPrefix(number)
hashes := make([]common.Hash, 0, 1)
it := db.NewIteratorWithPrefix(prefix)
it := db.NewIterator(prefix, nil)
defer it.Release()
for it.Next() {

120
core/rawdb/accessors_snapshot.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
// ReadSnapshotRoot retrieves the root of the block whose state is contained in
// the persisted snapshot.
func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash {
data, _ := db.Get(snapshotRootKey)
if len(data) != common.HashLength {
return common.Hash{}
}
return common.BytesToHash(data)
}
// WriteSnapshotRoot stores the root of the block whose state is contained in
// the persisted snapshot.
func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) {
if err := db.Put(snapshotRootKey, root[:]); err != nil {
log.Crit("Failed to store snapshot root", "err", err)
}
}
// DeleteSnapshotRoot deletes the hash of the block whose state is contained in
// the persisted snapshot. Since snapshots are not immutable, this method can
// be used during updates, so a crash or failure will mark the entire snapshot
// invalid.
func DeleteSnapshotRoot(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotRootKey); err != nil {
log.Crit("Failed to remove snapshot root", "err", err)
}
}
// ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf.
func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(accountSnapshotKey(hash))
return data
}
// WriteAccountSnapshot stores the snapshot entry of an account trie leaf.
func WriteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash, entry []byte) {
if err := db.Put(accountSnapshotKey(hash), entry); err != nil {
log.Crit("Failed to store account snapshot", "err", err)
}
}
// DeleteAccountSnapshot removes the snapshot entry of an account trie leaf.
func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(accountSnapshotKey(hash)); err != nil {
log.Crit("Failed to delete account snapshot", "err", err)
}
}
// ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf.
func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte {
data, _ := db.Get(storageSnapshotKey(accountHash, storageHash))
return data
}
// WriteStorageSnapshot stores the snapshot entry of an storage trie leaf.
func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) {
if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil {
log.Crit("Failed to store storage snapshot", "err", err)
}
}
// DeleteStorageSnapshot removes the snapshot entry of an storage trie leaf.
func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash) {
if err := db.Delete(storageSnapshotKey(accountHash, storageHash)); err != nil {
log.Crit("Failed to delete storage snapshot", "err", err)
}
}
// IterateStorageSnapshots returns an iterator for walking the entire storage
// space of a specific account.
func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator {
return db.NewIterator(storageSnapshotsKey(accountHash), nil)
}
// ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at
// the last shutdown. The blob is expected to be max a few 10s of megabytes.
func ReadSnapshotJournal(db ethdb.KeyValueReader) []byte {
data, _ := db.Get(snapshotJournalKey)
return data
}
// WriteSnapshotJournal stores the serialized in-memory diff layers to save at
// shutdown. The blob is expected to be max a few 10s of megabytes.
func WriteSnapshotJournal(db ethdb.KeyValueWriter, journal []byte) {
if err := db.Put(snapshotJournalKey, journal); err != nil {
log.Crit("Failed to store snapshot journal", "err", err)
}
}
// DeleteSnapshotJournal deletes the serialized in-memory diff layers saved at
// the last shutdown
func DeleteSnapshotJournal(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotJournalKey); err != nil {
log.Crit("Failed to remove snapshot journal", "err", err)
}
}

10
core/rawdb/database.go
View file

@ -221,7 +221,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database) error {
it := db.NewIterator()
it := db.NewIterator(nil, nil)
defer it.Release()
var (
@ -239,6 +239,8 @@ func InspectDatabase(db ethdb.Database) error {
hashNumPairing common.StorageSize
trieSize common.StorageSize
txlookupSize common.StorageSize
accountSnapSize common.StorageSize
storageSnapSize common.StorageSize
preimageSize common.StorageSize
bloomBitsSize common.StorageSize
cliqueSnapsSize common.StorageSize
@ -280,6 +282,10 @@ func InspectDatabase(db ethdb.Database) error {
receiptSize += size
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
txlookupSize += size
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
accountSnapSize += size
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
storageSnapSize += size
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
preimageSize += size
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
@ -331,6 +337,8 @@ func InspectDatabase(db ethdb.Database) error {
{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
{"Key-Value store", "Trie nodes", trieSize.String()},
{"Key-Value store", "Trie preimages", preimageSize.String()},
{"Key-Value store", "Account snapshot", accountSnapSize.String()},
{"Key-Value store", "Storage snapshot", storageSnapSize.String()},
{"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()},
{"Key-Value store", "Singleton metadata", metadata.String()},
{"Ancient store", "Headers", ancientHeaders.String()},

10
core/rawdb/freezer_table.go
View file

@ -541,20 +541,22 @@ func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) {
// Retrieve looks up the data offset of an item with the given number and retrieves
// the raw binary blob from the data file.
func (t *freezerTable) Retrieve(item uint64) ([]byte, error) {
t.lock.RLock()
// Ensure the table and the item is accessible
if t.index == nil || t.head == nil {
t.lock.RUnlock()
return nil, errClosed
}
if atomic.LoadUint64(&t.items) <= item {
t.lock.RUnlock()
return nil, errOutOfBounds
}
// Ensure the item was not deleted from the tail either
offset := atomic.LoadUint32(&t.itemOffset)
if uint64(offset) > item {
if uint64(t.itemOffset) > item {
t.lock.RUnlock()
return nil, errOutOfBounds
}
t.lock.RLock()
startOffset, endOffset, filenum, err := t.getBounds(item - uint64(offset))
startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset))
if err != nil {
t.lock.RUnlock()
return nil, err

6
core/rawdb/freezer_table_test.go
View file

@ -196,10 +196,8 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
f.Append(uint64(x), data)
}
// The last item should be there
if _, err = f.Retrieve(f.items - 1); err == nil {
if err != nil {
t.Fatal(err)
}
if _, err = f.Retrieve(f.items - 1); err != nil {
t.Fatal(err)
}
f.Close()
}

27
core/rawdb/schema.go
View file

@ -41,6 +41,12 @@ var (
// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
fastTrieProgressKey = []byte("TrieSync")
// snapshotRootKey tracks the hash of the last snapshot.
snapshotRootKey = []byte("SnapshotRoot")
// snapshotJournalKey tracks the in-memory diff layers across restarts.
snapshotJournalKey = []byte("SnapshotJournal")
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
@ -50,8 +56,10 @@ var (
blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body
blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts
txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value
SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value
preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
@ -145,6 +153,21 @@ func txLookupKey(hash common.Hash) []byte {
return append(txLookupPrefix, hash.Bytes()...)
}
// accountSnapshotKey = SnapshotAccountPrefix + hash
func accountSnapshotKey(hash common.Hash) []byte {
return append(SnapshotAccountPrefix, hash.Bytes()...)
}
// storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash
func storageSnapshotKey(accountHash, storageHash common.Hash) []byte {
return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...)
}
// storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash
func storageSnapshotsKey(accountHash common.Hash) []byte {
return append(SnapshotStoragePrefix, accountHash.Bytes()...)
}
// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...)

91
core/rawdb/table.go
View file

@ -103,23 +103,16 @@ func (t *table) Delete(key []byte) error {
return t.db.Delete(append([]byte(t.prefix), key...))
}
// NewIterator creates a binary-alphabetical iterator over the entire keyspace
// contained within the database.
func (t *table) NewIterator() ethdb.Iterator {
return t.NewIteratorWithPrefix(nil)
}
// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
// database content starting at a particular initial key (or after, if it does
// not exist).
func (t *table) NewIteratorWithStart(start []byte) ethdb.Iterator {
return t.db.NewIteratorWithStart(start)
}
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix.
func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
return t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...))
// NewIterator creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix, starting at a particular
// initial key (or after, if it does not exist).
func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
innerPrefix := append([]byte(t.prefix), prefix...)
iter := t.db.NewIterator(innerPrefix, start)
return &tableIterator{
iter: iter,
prefix: t.prefix,
}
}
// Stat returns a particular internal stat of the database.
@ -198,7 +191,69 @@ func (b *tableBatch) Reset() {
b.batch.Reset()
}
// tableReplayer is a wrapper around a batch replayer which truncates
// the added prefix.
type tableReplayer struct {
w ethdb.KeyValueWriter
prefix string
}
// Put implements the interface KeyValueWriter.
func (r *tableReplayer) Put(key []byte, value []byte) error {
trimmed := key[len(r.prefix):]
return r.w.Put(trimmed, value)
}
// Delete implements the interface KeyValueWriter.
func (r *tableReplayer) Delete(key []byte) error {
trimmed := key[len(r.prefix):]
return r.w.Delete(trimmed)
}
// Replay replays the batch contents.
func (b *tableBatch) Replay(w ethdb.KeyValueWriter) error {
return b.batch.Replay(w)
return b.batch.Replay(&tableReplayer{w: w, prefix: b.prefix})
}
// tableIterator is a wrapper around a database iterator that prefixes each key access
// with a pre-configured string.
type tableIterator struct {
iter ethdb.Iterator
prefix string
}
// Next moves the iterator to the next key/value pair. It returns whether the
// iterator is exhausted.
func (iter *tableIterator) Next() bool {
return iter.iter.Next()
}
// Error returns any accumulated error. Exhausting all the key/value pairs
// is not considered to be an error.
func (iter *tableIterator) Error() error {
return iter.iter.Error()
}
// Key returns the key of the current key/value pair, or nil if done. The caller
// should not modify the contents of the returned slice, and its contents may
// change on the next call to Next.
func (iter *tableIterator) Key() []byte {
key := iter.iter.Key()
if key == nil {
return nil
}
return key[len(iter.prefix):]
}
// Value returns the value of the current key/value pair, or nil if done. The
// caller should not modify the contents of the returned slice, and its contents
// may change on the next call to Next.
func (iter *tableIterator) Value() []byte {
return iter.iter.Value()
}
// Release releases associated resources. Release should always succeed and can
// be called multiple times without causing error.
func (iter *tableIterator) Release() {
iter.iter.Release()
}

128
core/rawdb/table_test.go Normal file
View file

@ -0,0 +1,128 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"bytes"
"testing"
"github.com/ethereum/go-ethereum/ethdb"
)
func TestTableDatabase(t *testing.T) { testTableDatabase(t, "prefix") }
func TestEmptyPrefixTableDatabase(t *testing.T) { testTableDatabase(t, "") }
type testReplayer struct {
puts [][]byte
dels [][]byte
}
func (r *testReplayer) Put(key []byte, value []byte) error {
r.puts = append(r.puts, key)
return nil
}
func (r *testReplayer) Delete(key []byte) error {
r.dels = append(r.dels, key)
return nil
}
func testTableDatabase(t *testing.T, prefix string) {
db := NewTable(NewMemoryDatabase(), prefix)
var entries = []struct {
key []byte
value []byte
}{
{[]byte{0x01, 0x02}, []byte{0x0a, 0x0b}},
{[]byte{0x03, 0x04}, []byte{0x0c, 0x0d}},
{[]byte{0x05, 0x06}, []byte{0x0e, 0x0f}},
{[]byte{0xff, 0xff, 0x01}, []byte{0x1a, 0x1b}},
{[]byte{0xff, 0xff, 0x02}, []byte{0x1c, 0x1d}},
{[]byte{0xff, 0xff, 0x03}, []byte{0x1e, 0x1f}},
}
// Test Put/Get operation
for _, entry := range entries {
db.Put(entry.key, entry.value)
}
for _, entry := range entries {
got, err := db.Get(entry.key)
if err != nil {
t.Fatalf("Failed to get value: %v", err)
}
if !bytes.Equal(got, entry.value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got)
}
}
// Test batch operation
db = NewTable(NewMemoryDatabase(), prefix)
batch := db.NewBatch()
for _, entry := range entries {
batch.Put(entry.key, entry.value)
}
batch.Write()
for _, entry := range entries {
got, err := db.Get(entry.key)
if err != nil {
t.Fatalf("Failed to get value: %v", err)
}
if !bytes.Equal(got, entry.value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got)
}
}
// Test batch replayer
r := &testReplayer{}
batch.Replay(r)
for index, entry := range entries {
got := r.puts[index]
if !bytes.Equal(got, entry.key) {
t.Fatalf("Key mismatch: want=%v, got=%v", entry.key, got)
}
}
check := func(iter ethdb.Iterator, expCount, index int) {
count := 0
for iter.Next() {
key, value := iter.Key(), iter.Value()
if !bytes.Equal(key, entries[index].key) {
t.Fatalf("Key mismatch: want=%v, got=%v", entries[index].key, key)
}
if !bytes.Equal(value, entries[index].value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entries[index].value, value)
}
index += 1
count++
}
if count != expCount {
t.Fatalf("Wrong number of elems, exp %d got %d", expCount, count)
}
iter.Release()
}
// Test iterators
check(db.NewIterator(nil, nil), 6, 0)
// Test iterators with prefix
check(db.NewIterator([]byte{0xff, 0xff}, nil), 3, 3)
// Test iterators with start point
check(db.NewIterator(nil, []byte{0xff, 0xff, 0x02}), 2, 4)
// Test iterators with prefix and start point
check(db.NewIterator([]byte{0xee}, nil), 0, 0)
check(db.NewIterator(nil, []byte{0x00}), 6, 0)
}

48
core/state/dump.go
View file

@ -27,7 +27,7 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
// DumpAccount represents an account in the state
// DumpAccount represents an account in the state.
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
@ -40,17 +40,24 @@ type DumpAccount struct {
}
// Dump represents the full dump in a collected format, as one large map
// Dump represents the full dump in a collected format, as one large map.
type Dump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
type iterativeDump struct {
*json.Encoder
}
// IteratorDump is an implementation for iterating over data.
type IteratorDump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
Next []byte `json:"next,omitempty"` // nil if no more accounts
}
// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d *IteratorDump) onRoot(root common.Hash) {
d.Root = fmt.Sprintf("%x", root)
}
func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
}{root})
}
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(s.trie.Hash())
it := trie.NewIterator(s.trie.NodeIterator(nil))
var count int
it := trie.NewIterator(s.trie.NodeIterator(start))
for it.Next() {
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
@ -130,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
}
}
c.onAccount(addr, account)
count++
if maxResults > 0 && count >= maxResults {
if it.Next() {
nextKey = it.Key
}
break
}
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
return nextKey
}
// RawDump returns the entire state an a single large object
@ -141,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
return *dump
}
@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
}
// IteratorDump dumps out a batch of accounts starts with the given start key
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
iterator := &IteratorDump{
Accounts: make(map[common.Address]DumpAccount),
}
iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
return *iterator
}

4
core/state/iterator_test.go
View file

@ -29,7 +29,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
// Create some arbitrary test state to iterate
db, root, _ := makeTestState()
state, err := New(root, db)
state, err := New(root, db, nil)
if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err)
}
@ -51,7 +51,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
t.Errorf("state entry not reported %x", hash)
}
}
it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator()
it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator(nil, nil)
for it.Next() {
key := it.Key()
if bytes.HasPrefix(key, []byte("secure-key-")) {

6
core/state/journal.go
View file

@ -90,7 +90,8 @@ type (
account *common.Address
}
resetObjectChange struct {
prev *stateObject
prev *stateObject
prevdestruct bool
}
suicideChange struct {
account *common.Address
@ -142,6 +143,9 @@ func (ch createObjectChange) dirtied() *common.Address {
func (ch resetObjectChange) revert(s *StateDB) {
s.setStateObject(ch.prev)
if !ch.prevdestruct && s.snap != nil {
delete(s.snapDestructs, ch.prev.addrHash)
}
}
func (ch resetObjectChange) dirtied() *common.Address {

54
core/state/snapshot/account.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
)
// Account is a slim version of a state.Account, where the root and code hash
// are replaced with a nil byte slice for empty accounts.
type Account struct {
Nonce uint64
Balance *big.Int
Root []byte
CodeHash []byte
}
// AccountRLP converts a state.Account content into a slim snapshot version RLP
// encoded.
func AccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte {
slim := Account{
Nonce: nonce,
Balance: balance,
}
if root != emptyRoot {
slim.Root = root[:]
}
if !bytes.Equal(codehash, emptyCode[:]) {
slim.CodeHash = codehash
}
data, err := rlp.EncodeToBytes(slim)
if err != nil {
panic(err)
}
return data
}

114
core/state/snapshot/conversion.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// conversionAccount is used for converting between full and slim format. When
// doing this, we can consider 'balance' as a byte array, as it has already
// been converted from big.Int into an rlp-byteslice.
type conversionAccount struct {
Nonce uint64
Balance []byte
Root []byte
CodeHash []byte
}
// SlimToFull converts data on the 'slim RLP' format into the full RLP-format
func SlimToFull(data []byte) ([]byte, error) {
acc := &conversionAccount{}
if err := rlp.DecodeBytes(data, acc); err != nil {
return nil, err
}
if len(acc.Root) == 0 {
acc.Root = emptyRoot[:]
}
if len(acc.CodeHash) == 0 {
acc.CodeHash = emptyCode[:]
}
fullData, err := rlp.EncodeToBytes(acc)
if err != nil {
return nil, err
}
return fullData, nil
}
// trieKV represents a trie key-value pair
type trieKV struct {
key common.Hash
value []byte
}
type trieGeneratorFn func(in chan (trieKV), out chan (common.Hash))
// GenerateTrieRoot takes an account iterator and reproduces the root hash.
func GenerateTrieRoot(it AccountIterator) common.Hash {
return generateTrieRoot(it, stdGenerate)
}
func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash {
var (
in = make(chan trieKV) // chan to pass leaves
out = make(chan common.Hash) // chan to collect result
wg sync.WaitGroup
)
wg.Add(1)
go func() {
generatorFn(in, out)
wg.Done()
}()
// Feed leaves
start := time.Now()
logged := time.Now()
accounts := 0
for it.Next() {
slimData := it.Account()
fullData, _ := SlimToFull(slimData)
l := trieKV{it.Hash(), fullData}
in <- l
if time.Since(logged) > 8*time.Second {
log.Info("Generating trie hash from snapshot",
"at", l.key, "accounts", accounts, "elapsed", time.Since(start))
logged = time.Now()
}
accounts++
}
close(in)
result := <-out
log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start))
wg.Wait()
return result
}
// stdGenerate is a very basic hexary trie builder which uses the same Trie
// as the rest of geth, with no enhancements or optimizations
func stdGenerate(in chan (trieKV), out chan (common.Hash)) {
t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New()))
for leaf := range in {
t.TryUpdate(leaf.key[:], leaf.value)
}
out <- t.Hash()
}

533
core/state/snapshot/difflayer.go Normal file
View file

@ -0,0 +1,533 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"encoding/binary"
"fmt"
"math"
"math/rand"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
"github.com/steakknife/bloomfilter"
)
var (
// aggregatorMemoryLimit is the maximum size of the bottom-most diff layer
// that aggregates the writes from above until it's flushed into the disk
// layer.
//
// Note, bumping this up might drastically increase the size of the bloom
// filters that's stored in every diff layer. Don't do that without fully
// understanding all the implications.
aggregatorMemoryLimit = uint64(4 * 1024 * 1024)
// aggregatorItemLimit is an approximate number of items that will end up
// in the agregator layer before it's flushed out to disk. A plain account
// weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot
// 0B (+hash). Slots are mostly set/unset in lockstep, so thet average at
// 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a
// smaller number to be on the safe side.
aggregatorItemLimit = aggregatorMemoryLimit / 42
// bloomTargetError is the target false positive rate when the aggregator
// layer is at its fullest. The actual value will probably move around up
// and down from this number, it's mostly a ballpark figure.
//
// Note, dropping this down might drastically increase the size of the bloom
// filters that's stored in every diff layer. Don't do that without fully
// understanding all the implications.
bloomTargetError = 0.02
// bloomSize is the ideal bloom filter size given the maximum number of items
// it's expected to hold and the target false positive error rate.
bloomSize = math.Ceil(float64(aggregatorItemLimit) * math.Log(bloomTargetError) / math.Log(1/math.Pow(2, math.Log(2))))
// bloomFuncs is the ideal number of bits a single entry should set in the
// bloom filter to keep its size to a minimum (given it's size and maximum
// entry count).
bloomFuncs = math.Round((bloomSize / float64(aggregatorItemLimit)) * math.Log(2))
// the bloom offsets are runtime constants which determines which part of the
// the account/storage hash the hasher functions looks at, to determine the
// bloom key for an account/slot. This is randomized at init(), so that the
// global population of nodes do not all display the exact same behaviour with
// regards to bloom content
bloomDestructHasherOffset = 0
bloomAccountHasherOffset = 0
bloomStorageHasherOffset = 0
)
func init() {
// Init the bloom offsets in the range [0:24] (requires 8 bytes)
bloomDestructHasherOffset = rand.Intn(25)
bloomAccountHasherOffset = rand.Intn(25)
bloomStorageHasherOffset = rand.Intn(25)
// The destruct and account blooms must be different, as the storage slots
// will check for destruction too for every bloom miss. It should not collide
// with modified accounts.
for bloomAccountHasherOffset == bloomDestructHasherOffset {
bloomAccountHasherOffset = rand.Intn(25)
}
}
// diffLayer represents a collection of modifications made to a state snapshot
// after running a block on top. It contains one sorted list for the account trie
// and one-one list for each storage tries.
//
// The goal of a diff layer is to act as a journal, tracking recent modifications
// made to the state, that have not yet graduated into a semi-immutable state.
type diffLayer struct {
origin *diskLayer // Base disk layer to directly use on bloom misses
parent snapshot // Parent snapshot modified by this one, never nil
memory uint64 // Approximate guess as to how much memory we use
root common.Hash // Root hash to which this snapshot diff belongs to
stale uint32 // Signals that the layer became stale (state progressed)
destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts
accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil
accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted)
storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrival. one per account (nil means deleted)
diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
lock sync.RWMutex
}
// destructBloomHasher is a wrapper around a common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert a destruct
// event into a 64 bit mini hash.
type destructBloomHasher common.Hash
func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h destructBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h destructBloomHasher) Reset() { panic("not implemented") }
func (h destructBloomHasher) BlockSize() int { panic("not implemented") }
func (h destructBloomHasher) Size() int { return 8 }
func (h destructBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8])
}
// accountBloomHasher is a wrapper around a common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert an account
// hash into a 64 bit mini hash.
type accountBloomHasher common.Hash
func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h accountBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h accountBloomHasher) Reset() { panic("not implemented") }
func (h accountBloomHasher) BlockSize() int { panic("not implemented") }
func (h accountBloomHasher) Size() int { return 8 }
func (h accountBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8])
}
// storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert an account
// hash into a 64 bit mini hash.
type storageBloomHasher [2]common.Hash
func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h storageBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h storageBloomHasher) Reset() { panic("not implemented") }
func (h storageBloomHasher) BlockSize() int { panic("not implemented") }
func (h storageBloomHasher) Size() int { return 8 }
func (h storageBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^
binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8])
}
// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
// level persistent database or a hierarchical diff already.
func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
// Create the new layer with some pre-allocated data segments
dl := &diffLayer{
parent: parent,
root: root,
destructSet: destructs,
accountData: accounts,
storageData: storage,
}
switch parent := parent.(type) {
case *diskLayer:
dl.rebloom(parent)
case *diffLayer:
dl.rebloom(parent.origin)
default:
panic("unknown parent type")
}
// Sanity check that accounts or storage slots are never nil
for accountHash, blob := range accounts {
if blob == nil {
panic(fmt.Sprintf("account %#x nil", accountHash))
}
}
for accountHash, slots := range storage {
if slots == nil {
panic(fmt.Sprintf("storage %#x nil", accountHash))
}
}
// Determine memory size and track the dirty writes
for _, data := range accounts {
dl.memory += uint64(common.HashLength + len(data))
snapshotDirtyAccountWriteMeter.Mark(int64(len(data)))
}
// Fill the storage hashes and sort them for the iterator
dl.storageList = make(map[common.Hash][]common.Hash)
for accountHash := range destructs {
dl.storageList[accountHash] = nil
}
// Determine memory size and track the dirty writes
for _, slots := range storage {
for _, data := range slots {
dl.memory += uint64(common.HashLength + len(data))
snapshotDirtyStorageWriteMeter.Mark(int64(len(data)))
}
}
dl.memory += uint64(len(dl.storageList) * common.HashLength)
return dl
}
// rebloom discards the layer's current bloom and rebuilds it from scratch based
// on the parent's and the local diffs.
func (dl *diffLayer) rebloom(origin *diskLayer) {
dl.lock.Lock()
defer dl.lock.Unlock()
defer func(start time.Time) {
snapshotBloomIndexTimer.Update(time.Since(start))
}(time.Now())
// Inject the new origin that triggered the rebloom
dl.origin = origin
// Retrieve the parent bloom or create a fresh empty one
if parent, ok := dl.parent.(*diffLayer); ok {
parent.lock.RLock()
dl.diffed, _ = parent.diffed.Copy()
parent.lock.RUnlock()
} else {
dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
}
// Iterate over all the accounts and storage slots and index them
for hash := range dl.destructSet {
dl.diffed.Add(destructBloomHasher(hash))
}
for hash := range dl.accountData {
dl.diffed.Add(accountBloomHasher(hash))
}
for accountHash, slots := range dl.storageData {
for storageHash := range slots {
dl.diffed.Add(storageBloomHasher{accountHash, storageHash})
}
}
// Calculate the current false positive rate and update the error rate meter.
// This is a bit cheating because subsequent layers will overwrite it, but it
// should be fine, we're only interested in ballpark figures.
k := float64(dl.diffed.K())
n := float64(dl.diffed.N())
m := float64(dl.diffed.M())
snapshotBloomErrorGauge.Update(math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k))
}
// Root returns the root hash for which this snapshot was made.
func (dl *diffLayer) Root() common.Hash {
return dl.root
}
// Parent returns the subsequent layer of a diff layer.
func (dl *diffLayer) Parent() snapshot {
return dl.parent
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diffLayer) Stale() bool {
return atomic.LoadUint32(&dl.stale) != 0
}
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
data, err := dl.AccountRLP(hash)
if err != nil {
return nil, err
}
if len(data) == 0 { // can be both nil and []byte{}
return nil, nil
}
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
return account, nil
}
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
// Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below
dl.lock.RLock()
hit := dl.diffed.Contains(accountBloomHasher(hash))
if !hit {
hit = dl.diffed.Contains(destructBloomHasher(hash))
}
dl.lock.RUnlock()
// If the bloom filter misses, don't even bother with traversing the memory
// diff layers, reach straight into the bottom persistent disk layer
if !hit {
snapshotBloomAccountMissMeter.Mark(1)
return dl.origin.AccountRLP(hash)
}
// The bloom filter hit, start poking in the internal maps
return dl.accountRLP(hash, 0)
}
// accountRLP is an internal version of AccountRLP that skips the bloom filter
// checks and uses the internal maps to try and retrieve the data. It's meant
// to be used if a higher layer's bloom filter hit already.
func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.Stale() {
return nil, ErrSnapshotStale
}
// If the account is known locally, return it
if data, ok := dl.accountData[hash]; ok {
snapshotDirtyAccountHitMeter.Mark(1)
snapshotDirtyAccountHitDepthHist.Update(int64(depth))
snapshotDirtyAccountReadMeter.Mark(int64(len(data)))
snapshotBloomAccountTrueHitMeter.Mark(1)
return data, nil
}
// If the account is known locally, but deleted, return it
if _, ok := dl.destructSet[hash]; ok {
snapshotDirtyAccountHitMeter.Mark(1)
snapshotDirtyAccountHitDepthHist.Update(int64(depth))
snapshotDirtyAccountInexMeter.Mark(1)
snapshotBloomAccountTrueHitMeter.Mark(1)
return nil, nil
}
// Account unknown to this diff, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.accountRLP(hash, depth+1)
}
// Failed to resolve through diff layers, mark a bloom error and use the disk
snapshotBloomAccountFalseHitMeter.Mark(1)
return dl.parent.AccountRLP(hash)
}
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account. If the slot is unknown to this diff, it's parent
// is consulted.
func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
// Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below
dl.lock.RLock()
hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash})
if !hit {
hit = dl.diffed.Contains(destructBloomHasher(accountHash))
}
dl.lock.RUnlock()
// If the bloom filter misses, don't even bother with traversing the memory
// diff layers, reach straight into the bottom persistent disk layer
if !hit {
snapshotBloomStorageMissMeter.Mark(1)
return dl.origin.Storage(accountHash, storageHash)
}
// The bloom filter hit, start poking in the internal maps
return dl.storage(accountHash, storageHash, 0)
}
// storage is an internal version of Storage that skips the bloom filter checks
// and uses the internal maps to try and retrieve the data. It's meant to be
// used if a higher layer's bloom filter hit already.
func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.Stale() {
return nil, ErrSnapshotStale
}
// If the account is known locally, try to resolve the slot locally
if storage, ok := dl.storageData[accountHash]; ok {
if data, ok := storage[storageHash]; ok {
snapshotDirtyStorageHitMeter.Mark(1)
snapshotDirtyStorageHitDepthHist.Update(int64(depth))
if n := len(data); n > 0 {
snapshotDirtyStorageReadMeter.Mark(int64(n))
} else {
snapshotDirtyStorageInexMeter.Mark(1)
}
snapshotBloomStorageTrueHitMeter.Mark(1)
return data, nil
}
}
// If the account is known locally, but deleted, return an empty slot
if _, ok := dl.destructSet[accountHash]; ok {
snapshotDirtyStorageHitMeter.Mark(1)
snapshotDirtyStorageHitDepthHist.Update(int64(depth))
snapshotDirtyStorageInexMeter.Mark(1)
snapshotBloomStorageTrueHitMeter.Mark(1)
return nil, nil
}
// Storage slot unknown to this diff, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.storage(accountHash, storageHash, depth+1)
}
// Failed to resolve through diff layers, mark a bloom error and use the disk
snapshotBloomStorageFalseHitMeter.Mark(1)
return dl.parent.Storage(accountHash, storageHash)
}
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items.
func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, blockRoot, destructs, accounts, storage)
}
// flatten pushes all data from this point downwards, flattening everything into
// a single diff at the bottom. Since usually the lowermost diff is the largest,
// the flattening bulds up from there in reverse.
func (dl *diffLayer) flatten() snapshot {
// If the parent is not diff, we're the first in line, return unmodified
parent, ok := dl.parent.(*diffLayer)
if !ok {
return dl
}
// Parent is a diff, flatten it first (note, apart from weird corned cases,
// flatten will realistically only ever merge 1 layer, so there's no need to
// be smarter about grouping flattens together).
parent = parent.flatten().(*diffLayer)
parent.lock.Lock()
defer parent.lock.Unlock()
// Before actually writing all our data to the parent, first ensure that the
// parent hasn't been 'corrupted' by someone else already flattening into it
if atomic.SwapUint32(&parent.stale, 1) != 0 {
panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo
}
// Overwrite all the updated accounts blindly, merge the sorted list
for hash := range dl.destructSet {
parent.destructSet[hash] = struct{}{}
delete(parent.accountData, hash)
delete(parent.storageData, hash)
}
for hash, data := range dl.accountData {
parent.accountData[hash] = data
}
// Overwrite all the updated storage slots (individually)
for accountHash, storage := range dl.storageData {
// If storage didn't exist (or was deleted) in the parent, overwrite blindly
if _, ok := parent.storageData[accountHash]; !ok {
parent.storageData[accountHash] = storage
continue
}
// Storage exists in both parent and child, merge the slots
comboData := parent.storageData[accountHash]
for storageHash, data := range storage {
comboData[storageHash] = data
}
parent.storageData[accountHash] = comboData
}
// Return the combo parent
return &diffLayer{
parent: parent.parent,
origin: parent.origin,
root: dl.root,
destructSet: parent.destructSet,
accountData: parent.accountData,
storageData: parent.storageData,
storageList: make(map[common.Hash][]common.Hash),
diffed: dl.diffed,
memory: parent.memory + dl.memory,
}
}
// AccountList returns a sorted list of all accounts in this difflayer, including
// the deleted ones.
//
// Note, the returned slice is not a copy, so do not modify it.
func (dl *diffLayer) AccountList() []common.Hash {
// If an old list already exists, return it
dl.lock.RLock()
list := dl.accountList
dl.lock.RUnlock()
if list != nil {
return list
}
// No old sorted account list exists, generate a new one
dl.lock.Lock()
defer dl.lock.Unlock()
dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData))
for hash := range dl.accountData {
dl.accountList = append(dl.accountList, hash)
}
for hash := range dl.destructSet {
if _, ok := dl.accountData[hash]; !ok {
dl.accountList = append(dl.accountList, hash)
}
}
sort.Sort(hashes(dl.accountList))
return dl.accountList
}
// StorageList returns a sorted list of all storage slot hashes in this difflayer
// for the given account.
//
// Note, the returned slice is not a copy, so do not modify it.
func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash {
// If an old list already exists, return it
dl.lock.RLock()
list := dl.storageList[accountHash]
dl.lock.RUnlock()
if list != nil {
return list
}
// No old sorted account list exists, generate a new one
dl.lock.Lock()
defer dl.lock.Unlock()
storageMap := dl.storageData[accountHash]
storageList := make([]common.Hash, 0, len(storageMap))
for k := range storageMap {
storageList = append(storageList, k)
}
sort.Sort(hashes(storageList))
dl.storageList[accountHash] = storageList
return storageList
}

399
core/state/snapshot/difflayer_test.go Normal file
View file

@ -0,0 +1,399 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"math/rand"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} {
copy := make(map[common.Hash]struct{})
for hash := range destructs {
copy[hash] = struct{}{}
}
return copy
}
func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte {
copy := make(map[common.Hash][]byte)
for hash, blob := range accounts {
copy[hash] = blob
}
return copy
}
func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte {
copy := make(map[common.Hash]map[common.Hash][]byte)
for accHash, slots := range storage {
copy[accHash] = make(map[common.Hash][]byte)
for slotHash, blob := range slots {
copy[accHash][slotHash] = blob
}
}
return copy
}
// TestMergeBasics tests some simple merges
func TestMergeBasics(t *testing.T) {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
// Fill up a parent
for i := 0; i < 100; i++ {
h := randomHash()
data := randomAccount()
accounts[h] = data
if rand.Intn(4) == 0 {
destructs[h] = struct{}{}
}
if rand.Intn(2) == 0 {
accStorage := make(map[common.Hash][]byte)
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
storage[h] = accStorage
}
}
// Add some (identical) layers on top
parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
// And flatten
merged := (child.flatten()).(*diffLayer)
{ // Check account lists
if have, want := len(merged.accountList), 0; have != want {
t.Errorf("accountList wrong: have %v, want %v", have, want)
}
if have, want := len(merged.AccountList()), len(accounts); have != want {
t.Errorf("AccountList() wrong: have %v, want %v", have, want)
}
if have, want := len(merged.accountList), len(accounts); have != want {
t.Errorf("accountList [2] wrong: have %v, want %v", have, want)
}
}
{ // Check account drops
if have, want := len(merged.destructSet), len(destructs); have != want {
t.Errorf("accountDrop wrong: have %v, want %v", have, want)
}
}
{ // Check storage lists
i := 0
for aHash, sMap := range storage {
if have, want := len(merged.storageList), i; have != want {
t.Errorf("[1] storageList wrong: have %v, want %v", have, want)
}
if have, want := len(merged.StorageList(aHash)), len(sMap); have != want {
t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want)
}
if have, want := len(merged.storageList[aHash]), len(sMap); have != want {
t.Errorf("storageList wrong: have %v, want %v", have, want)
}
i++
}
}
}
// TestMergeDelete tests some deletion
func TestMergeDelete(t *testing.T) {
var (
storage = make(map[common.Hash]map[common.Hash][]byte)
)
// Fill up a parent
h1 := common.HexToHash("0x01")
h2 := common.HexToHash("0x02")
flipDrops := func() map[common.Hash]struct{} {
return map[common.Hash]struct{}{
h2: struct{}{},
}
}
flipAccs := func() map[common.Hash][]byte {
return map[common.Hash][]byte{
h1: randomAccount(),
}
}
flopDrops := func() map[common.Hash]struct{} {
return map[common.Hash]struct{}{
h1: struct{}{},
}
}
flopAccs := func() map[common.Hash][]byte {
return map[common.Hash][]byte{
h2: randomAccount(),
}
}
// Add some flipAccs-flopping layers on top
parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage)
child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
if data, _ := child.Account(h1); data == nil {
t.Errorf("last diff layer: expected %x account to be non-nil", h1)
}
if data, _ := child.Account(h2); data != nil {
t.Errorf("last diff layer: expected %x account to be nil", h2)
}
if _, ok := child.destructSet[h1]; ok {
t.Errorf("last diff layer: expected %x drop to be missing", h1)
}
if _, ok := child.destructSet[h2]; !ok {
t.Errorf("last diff layer: expected %x drop to be present", h1)
}
// And flatten
merged := (child.flatten()).(*diffLayer)
if data, _ := merged.Account(h1); data == nil {
t.Errorf("merged layer: expected %x account to be non-nil", h1)
}
if data, _ := merged.Account(h2); data != nil {
t.Errorf("merged layer: expected %x account to be nil", h2)
}
if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk!
t.Errorf("merged diff layer: expected %x drop to be present", h1)
}
if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk!
t.Errorf("merged diff layer: expected %x drop to be present", h1)
}
// If we add more granular metering of memory, we can enable this again,
// but it's not implemented for now
//if have, want := merged.memory, child.memory; have != want {
// t.Errorf("mem wrong: have %d, want %d", have, want)
//}
}
// This tests that if we create a new account, and set a slot, and then merge
// it, the lists will be correct.
func TestInsertAndMerge(t *testing.T) {
// Fill up a parent
var (
acc = common.HexToHash("0x01")
slot = common.HexToHash("0x02")
parent *diffLayer
child *diffLayer
)
{
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage)
}
{
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
accounts[acc] = randomAccount()
storage[acc] = make(map[common.Hash][]byte)
storage[acc][slot] = []byte{0x01}
child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
// And flatten
merged := (child.flatten()).(*diffLayer)
{ // Check that slot value is present
have, _ := merged.Storage(acc, slot)
if want := []byte{0x01}; !bytes.Equal(have, want) {
t.Errorf("merged slot value wrong: have %x, want %x", have, want)
}
}
}
func emptyLayer() *diskLayer {
return &diskLayer{
diskdb: memorydb.New(),
cache: fastcache.New(500 * 1024),
}
}
// BenchmarkSearch checks how long it takes to find a non-existing key
// BenchmarkSearch-6 200000 10481 ns/op (1K per layer)
// BenchmarkSearch-6 200000 10760 ns/op (10K per layer)
// BenchmarkSearch-6 100000 17866 ns/op
//
// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock()
func BenchmarkSearch(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 10000; i++ {
accounts[randomHash()] = randomAccount()
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
var layer snapshot
layer = emptyLayer()
for i := 0; i < 128; i++ {
layer = fill(layer)
}
key := crypto.Keccak256Hash([]byte{0x13, 0x38})
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.AccountRLP(key)
}
}
// BenchmarkSearchSlot checks how long it takes to find a non-existing key
// - Number of layers: 128
// - Each layers contains the account, with a couple of storage slots
// BenchmarkSearchSlot-6 100000 14554 ns/op
// BenchmarkSearchSlot-6 100000 22254 ns/op (when checking parent root using mutex)
// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic)
// With bloom filter:
// BenchmarkSearchSlot-6 3467835 351 ns/op
func BenchmarkSearchSlot(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each
accountKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
storageKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
accountRLP := randomAccount()
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
accounts[accountKey] = accountRLP
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 5; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
var layer snapshot
layer = emptyLayer()
for i := 0; i < 128; i++ {
layer = fill(layer)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.Storage(accountKey, storageKey)
}
}
// With accountList and sorting
// BenchmarkFlatten-6 50 29890856 ns/op
//
// Without sorting and tracking accountlist
// BenchmarkFlatten-6 300 5511511 ns/op
func BenchmarkFlatten(b *testing.B) {
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 100; i++ {
accountKey := randomHash()
accounts[accountKey] = randomAccount()
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 20; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
}
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
var layer snapshot
layer = emptyLayer()
for i := 1; i < 128; i++ {
layer = fill(layer)
}
b.StartTimer()
for i := 1; i < 128; i++ {
dl, ok := layer.(*diffLayer)
if !ok {
break
}
layer = dl.flatten()
}
b.StopTimer()
}
}
// This test writes ~324M of diff layers to disk, spread over
// - 128 individual layers,
// - each with 200 accounts
// - containing 200 slots
//
// BenchmarkJournal-6 1 1471373923 ns/ops
// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer
func BenchmarkJournal(b *testing.B) {
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 200; i++ {
accountKey := randomHash()
accounts[accountKey] = randomAccount()
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 200; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
}
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
layer := snapshot(new(diskLayer))
for i := 1; i < 128; i++ {
layer = fill(layer)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.Journal(new(bytes.Buffer))
}
}

166
core/state/snapshot/disklayer.go Normal file
View file

@ -0,0 +1,166 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"sync"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// diskLayer is a low level persistent snapshot built on top of a key-value store.
type diskLayer struct {
diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot
triedb *trie.Database // Trie node cache for reconstuction purposes
cache *fastcache.Cache // Cache to avoid hitting the disk for direct access
root common.Hash // Root hash of the base snapshot
stale bool // Signals that the layer became stale (state progressed)
genMarker []byte // Marker for the state that's indexed during initial layer generation
genPending chan struct{} // Notification channel when generation is done (test synchronicity)
genAbort chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer
lock sync.RWMutex
}
// Root returns root hash for which this snapshot was made.
func (dl *diskLayer) Root() common.Hash {
return dl.root
}
// Parent always returns nil as there's no layer below the disk.
func (dl *diskLayer) Parent() snapshot {
return nil
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diskLayer) Stale() bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.stale
}
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
func (dl *diskLayer) Account(hash common.Hash) (*Account, error) {
data, err := dl.AccountRLP(hash)
if err != nil {
return nil, err
}
if len(data) == 0 { // can be both nil and []byte{}
return nil, nil
}
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
return account, nil
}
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
// If the layer is being generated, ensure the requested hash has already been
// covered by the generator.
if dl.genMarker != nil && bytes.Compare(hash[:], dl.genMarker) > 0 {
return nil, ErrNotCoveredYet
}
// If we're in the disk layer, all diff layers missed
snapshotDirtyAccountMissMeter.Mark(1)
// Try to retrieve the account from the memory cache
if blob, found := dl.cache.HasGet(nil, hash[:]); found {
snapshotCleanAccountHitMeter.Mark(1)
snapshotCleanAccountReadMeter.Mark(int64(len(blob)))
return blob, nil
}
// Cache doesn't contain account, pull from disk and cache for later
blob := rawdb.ReadAccountSnapshot(dl.diskdb, hash)
dl.cache.Set(hash[:], blob)
snapshotCleanAccountMissMeter.Mark(1)
if n := len(blob); n > 0 {
snapshotCleanAccountWriteMeter.Mark(int64(n))
} else {
snapshotCleanAccountInexMeter.Mark(1)
}
return blob, nil
}
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account.
func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
key := append(accountHash[:], storageHash[:]...)
// If the layer is being generated, ensure the requested hash has already been
// covered by the generator.
if dl.genMarker != nil && bytes.Compare(key, dl.genMarker) > 0 {
return nil, ErrNotCoveredYet
}
// If we're in the disk layer, all diff layers missed
snapshotDirtyStorageMissMeter.Mark(1)
// Try to retrieve the storage slot from the memory cache
if blob, found := dl.cache.HasGet(nil, key); found {
snapshotCleanStorageHitMeter.Mark(1)
snapshotCleanStorageReadMeter.Mark(int64(len(blob)))
return blob, nil
}
// Cache doesn't contain storage slot, pull from disk and cache for later
blob := rawdb.ReadStorageSnapshot(dl.diskdb, accountHash, storageHash)
dl.cache.Set(key, blob)
snapshotCleanStorageMissMeter.Mark(1)
if n := len(blob); n > 0 {
snapshotCleanStorageWriteMeter.Mark(int64(n))
} else {
snapshotCleanStorageInexMeter.Mark(1)
}
return blob, nil
}
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items. Note, the maps are retained by the method to avoid
// copying everything.
func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, blockHash, destructs, accounts, storage)
}

511
core/state/snapshot/disklayer_test.go Normal file
View file

@ -0,0 +1,511 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
// reverse reverses the contents of a byte slice. It's used to update random accs
// with deterministic changes.
func reverse(blob []byte) []byte {
res := make([]byte, len(blob))
for i, b := range blob {
res[len(blob)-1-i] = b
}
return res
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values.
func TestDiskMerge(t *testing.T) {
// Create some accounts in the disk layer
db := memorydb.New()
var (
accNoModNoCache = common.Hash{0x1}
accNoModCache = common.Hash{0x2}
accModNoCache = common.Hash{0x3}
accModCache = common.Hash{0x4}
accDelNoCache = common.Hash{0x5}
accDelCache = common.Hash{0x6}
conNoModNoCache = common.Hash{0x7}
conNoModNoCacheSlot = common.Hash{0x70}
conNoModCache = common.Hash{0x8}
conNoModCacheSlot = common.Hash{0x80}
conModNoCache = common.Hash{0x9}
conModNoCacheSlot = common.Hash{0x90}
conModCache = common.Hash{0xa}
conModCacheSlot = common.Hash{0xa0}
conDelNoCache = common.Hash{0xb}
conDelNoCacheSlot = common.Hash{0xb0}
conDelCache = common.Hash{0xc}
conDelCacheSlot = common.Hash{0xc0}
conNukeNoCache = common.Hash{0xd}
conNukeNoCacheSlot = common.Hash{0xd0}
conNukeCache = common.Hash{0xe}
conNukeCacheSlot = common.Hash{0xe0}
baseRoot = randomHash()
diffRoot = randomHash()
)
rawdb.WriteAccountSnapshot(db, accNoModNoCache, accNoModNoCache[:])
rawdb.WriteAccountSnapshot(db, accNoModCache, accNoModCache[:])
rawdb.WriteAccountSnapshot(db, accModNoCache, accModNoCache[:])
rawdb.WriteAccountSnapshot(db, accModCache, accModCache[:])
rawdb.WriteAccountSnapshot(db, accDelNoCache, accDelNoCache[:])
rawdb.WriteAccountSnapshot(db, accDelCache, accDelCache[:])
rawdb.WriteAccountSnapshot(db, conNoModNoCache, conNoModNoCache[:])
rawdb.WriteStorageSnapshot(db, conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNoModCache, conNoModCache[:])
rawdb.WriteStorageSnapshot(db, conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conModNoCache, conModNoCache[:])
rawdb.WriteStorageSnapshot(db, conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conModCache, conModCache[:])
rawdb.WriteStorageSnapshot(db, conModCache, conModCacheSlot, conModCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conDelNoCache, conDelNoCache[:])
rawdb.WriteStorageSnapshot(db, conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conDelCache, conDelCache[:])
rawdb.WriteStorageSnapshot(db, conDelCache, conDelCacheSlot, conDelCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNukeNoCache, conNukeNoCache[:])
rawdb.WriteStorageSnapshot(db, conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNukeCache, conNukeCache[:])
rawdb.WriteStorageSnapshot(db, conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
rawdb.WriteSnapshotRoot(db, baseRoot)
// Create a disk layer based on the above and cache in some data
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
base := snaps.Snapshot(baseRoot)
base.AccountRLP(accNoModCache)
base.AccountRLP(accModCache)
base.AccountRLP(accDelCache)
base.Storage(conNoModCache, conNoModCacheSlot)
base.Storage(conModCache, conModCacheSlot)
base.Storage(conDelCache, conDelCacheSlot)
base.Storage(conNukeCache, conNukeCacheSlot)
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{
accDelNoCache: struct{}{},
accDelCache: struct{}{},
conNukeNoCache: struct{}{},
conNukeCache: struct{}{},
}, map[common.Hash][]byte{
accModNoCache: reverse(accModNoCache[:]),
accModCache: reverse(accModCache[:]),
}, map[common.Hash]map[common.Hash][]byte{
conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])},
conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])},
conDelNoCache: {conDelNoCacheSlot: nil},
conDelCache: {conDelCacheSlot: nil},
}); err != nil {
t.Fatalf("failed to update snapshot tree: %v", err)
}
if err := snaps.Cap(diffRoot, 0); err != nil {
t.Fatalf("failed to flatten snapshot tree: %v", err)
}
// Retrieve all the data through the disk layer and validate it
base = snaps.Snapshot(diffRoot)
if _, ok := base.(*diskLayer); !ok {
t.Fatalf("update not flattend into the disk layer")
}
// assertAccount ensures that an account matches the given blob.
assertAccount := func(account common.Hash, data []byte) {
t.Helper()
blob, err := base.AccountRLP(account)
if err != nil {
t.Errorf("account access (%x) failed: %v", account, err)
} else if !bytes.Equal(blob, data) {
t.Errorf("account access (%x) mismatch: have %x, want %x", account, blob, data)
}
}
assertAccount(accNoModNoCache, accNoModNoCache[:])
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModNoCache, reverse(accModNoCache[:]))
assertAccount(accModCache, reverse(accModCache[:]))
assertAccount(accDelNoCache, nil)
assertAccount(accDelCache, nil)
// assertStorage ensures that a storage slot matches the given blob.
assertStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob, err := base.Storage(account, slot)
if err != nil {
t.Errorf("storage access (%x:%x) failed: %v", account, slot, err)
} else if !bytes.Equal(blob, data) {
t.Errorf("storage access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data)
}
}
assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertStorage(conDelCache, conDelCacheSlot, nil)
assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertStorage(conNukeCache, conNukeCacheSlot, nil)
// Retrieve all the data directly from the database and validate it
// assertDatabaseAccount ensures that an account from the database matches the given blob.
assertDatabaseAccount := func(account common.Hash, data []byte) {
t.Helper()
if blob := rawdb.ReadAccountSnapshot(db, account); !bytes.Equal(blob, data) {
t.Errorf("account database access (%x) mismatch: have %x, want %x", account, blob, data)
}
}
assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:])
assertDatabaseAccount(accNoModCache, accNoModCache[:])
assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:]))
assertDatabaseAccount(accModCache, reverse(accModCache[:]))
assertDatabaseAccount(accDelNoCache, nil)
assertDatabaseAccount(accDelCache, nil)
// assertDatabaseStorage ensures that a storage slot from the database matches the given blob.
assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
if blob := rawdb.ReadStorageSnapshot(db, account, slot); !bytes.Equal(blob, data) {
t.Errorf("storage database access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data)
}
}
assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertDatabaseStorage(conDelCache, conDelCacheSlot, nil)
assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil)
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values, discarding anything
// after the in-progress generation marker.
func TestDiskPartialMerge(t *testing.T) {
// Iterate the test a few times to ensure we pick various internal orderings
// for the data slots as well as the progress marker.
for i := 0; i < 1024; i++ {
// Create some accounts in the disk layer
db := memorydb.New()
var (
accNoModNoCache = randomHash()
accNoModCache = randomHash()
accModNoCache = randomHash()
accModCache = randomHash()
accDelNoCache = randomHash()
accDelCache = randomHash()
conNoModNoCache = randomHash()
conNoModNoCacheSlot = randomHash()
conNoModCache = randomHash()
conNoModCacheSlot = randomHash()
conModNoCache = randomHash()
conModNoCacheSlot = randomHash()
conModCache = randomHash()
conModCacheSlot = randomHash()
conDelNoCache = randomHash()
conDelNoCacheSlot = randomHash()
conDelCache = randomHash()
conDelCacheSlot = randomHash()
conNukeNoCache = randomHash()
conNukeNoCacheSlot = randomHash()
conNukeCache = randomHash()
conNukeCacheSlot = randomHash()
baseRoot = randomHash()
diffRoot = randomHash()
genMarker = append(randomHash().Bytes(), randomHash().Bytes()...)
)
// insertAccount injects an account into the database if it's after the
// generator marker, drops the op otherwise. This is needed to seed the
// database with a valid starting snapshot.
insertAccount := func(account common.Hash, data []byte) {
if bytes.Compare(account[:], genMarker) <= 0 {
rawdb.WriteAccountSnapshot(db, account, data[:])
}
}
insertAccount(accNoModNoCache, accNoModNoCache[:])
insertAccount(accNoModCache, accNoModCache[:])
insertAccount(accModNoCache, accModNoCache[:])
insertAccount(accModCache, accModCache[:])
insertAccount(accDelNoCache, accDelNoCache[:])
insertAccount(accDelCache, accDelCache[:])
// insertStorage injects a storage slot into the database if it's after
// the generator marker, drops the op otherwise. This is needed to seed
// the database with a valid starting snapshot.
insertStorage := func(account common.Hash, slot common.Hash, data []byte) {
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 {
rawdb.WriteStorageSnapshot(db, account, slot, data[:])
}
}
insertAccount(conNoModNoCache, conNoModNoCache[:])
insertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
insertAccount(conNoModCache, conNoModCache[:])
insertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
insertAccount(conModNoCache, conModNoCache[:])
insertStorage(conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:])
insertAccount(conModCache, conModCache[:])
insertStorage(conModCache, conModCacheSlot, conModCacheSlot[:])
insertAccount(conDelNoCache, conDelNoCache[:])
insertStorage(conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:])
insertAccount(conDelCache, conDelCache[:])
insertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:])
insertAccount(conNukeNoCache, conNukeNoCache[:])
insertStorage(conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:])
insertAccount(conNukeCache, conNukeCache[:])
insertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
rawdb.WriteSnapshotRoot(db, baseRoot)
// Create a disk layer based on the above using a random progress marker
// and cache in some data.
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
snaps.layers[baseRoot].(*diskLayer).genMarker = genMarker
base := snaps.Snapshot(baseRoot)
// assertAccount ensures that an account matches the given blob if it's
// already covered by the disk snapshot, and errors out otherwise.
assertAccount := func(account common.Hash, data []byte) {
t.Helper()
blob, err := base.AccountRLP(account)
if bytes.Compare(account[:], genMarker) > 0 && err != ErrNotCoveredYet {
t.Fatalf("test %d: post-marker (%x) account access (%x) succeeded: %x", i, genMarker, account, blob)
}
if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) account access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data)
}
}
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModCache, accModCache[:])
assertAccount(accDelCache, accDelCache[:])
// assertStorage ensures that a storage slot matches the given blob if
// it's already covered by the disk snapshot, and errors out otherwise.
assertStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob, err := base.Storage(account, slot)
if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && err != ErrNotCoveredYet {
t.Fatalf("test %d: post-marker (%x) storage access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob)
}
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) storage access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data)
}
}
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModCache, conModCacheSlot, conModCacheSlot[:])
assertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:])
assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{
accDelNoCache: struct{}{},
accDelCache: struct{}{},
conNukeNoCache: struct{}{},
conNukeCache: struct{}{},
}, map[common.Hash][]byte{
accModNoCache: reverse(accModNoCache[:]),
accModCache: reverse(accModCache[:]),
}, map[common.Hash]map[common.Hash][]byte{
conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])},
conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])},
conDelNoCache: {conDelNoCacheSlot: nil},
conDelCache: {conDelCacheSlot: nil},
}); err != nil {
t.Fatalf("test %d: failed to update snapshot tree: %v", i, err)
}
if err := snaps.Cap(diffRoot, 0); err != nil {
t.Fatalf("test %d: failed to flatten snapshot tree: %v", i, err)
}
// Retrieve all the data through the disk layer and validate it
base = snaps.Snapshot(diffRoot)
if _, ok := base.(*diskLayer); !ok {
t.Fatalf("test %d: update not flattend into the disk layer", i)
}
assertAccount(accNoModNoCache, accNoModNoCache[:])
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModNoCache, reverse(accModNoCache[:]))
assertAccount(accModCache, reverse(accModCache[:]))
assertAccount(accDelNoCache, nil)
assertAccount(accDelCache, nil)
assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertStorage(conDelCache, conDelCacheSlot, nil)
assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertStorage(conNukeCache, conNukeCacheSlot, nil)
// Retrieve all the data directly from the database and validate it
// assertDatabaseAccount ensures that an account inside the database matches
// the given blob if it's already covered by the disk snapshot, and does not
// exist otherwise.
assertDatabaseAccount := func(account common.Hash, data []byte) {
t.Helper()
blob := rawdb.ReadAccountSnapshot(db, account)
if bytes.Compare(account[:], genMarker) > 0 && blob != nil {
t.Fatalf("test %d: post-marker (%x) account database access (%x) succeeded: %x", i, genMarker, account, blob)
}
if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) account database access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data)
}
}
assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:])
assertDatabaseAccount(accNoModCache, accNoModCache[:])
assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:]))
assertDatabaseAccount(accModCache, reverse(accModCache[:]))
assertDatabaseAccount(accDelNoCache, nil)
assertDatabaseAccount(accDelCache, nil)
// assertDatabaseStorage ensures that a storage slot inside the database
// matches the given blob if it's already covered by the disk snapshot,
// and does not exist otherwise.
assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob := rawdb.ReadStorageSnapshot(db, account, slot)
if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && blob != nil {
t.Fatalf("test %d: post-marker (%x) storage database access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob)
}
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) storage database access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data)
}
}
assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertDatabaseStorage(conDelCache, conDelCacheSlot, nil)
assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil)
}
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values, discarding anything
// after the in-progress generation marker.
//
// This test case is a tiny specialized case of TestDiskPartialMerge, which tests
// some very specific cornercases that random tests won't ever trigger.
func TestDiskMidAccountPartialMerge(t *testing.T) {
// TODO(@karalabe) ?
}
// TestDiskSeek tests that seek-operations work on the disk layer
func TestDiskSeek(t *testing.T) {
// Create some accounts in the disk layer
var db ethdb.Database
if dir, err := ioutil.TempDir("", "disklayer-test"); err != nil {
t.Fatal(err)
} else {
defer os.RemoveAll(dir)
diskdb, err := leveldb.New(dir, 256, 0, "")
if err != nil {
t.Fatal(err)
}
db = rawdb.NewDatabase(diskdb)
}
// Fill even keys [0,2,4...]
for i := 0; i < 0xff; i += 2 {
acc := common.Hash{byte(i)}
rawdb.WriteAccountSnapshot(db, acc, acc[:])
}
// Add an 'higher' key, with incorrect (higher) prefix
highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1}
db.Put(highKey, []byte{0xff, 0xff})
baseRoot := randomHash()
rawdb.WriteSnapshotRoot(db, baseRoot)
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
// Test some different seek positions
type testcase struct {
pos byte
expkey byte
}
var cases = []testcase{
{0xff, 0x55}, // this should exit immediately without checking key
{0x01, 0x02},
{0xfe, 0xfe},
{0xfd, 0xfe},
{0x00, 0x00},
}
for i, tc := range cases {
it, err := snaps.AccountIterator(baseRoot, common.Hash{tc.pos})
if err != nil {
t.Fatalf("case %d, error: %v", i, err)
}
count := 0
for it.Next() {
k, v, err := it.Hash()[0], it.Account()[0], it.Error()
if err != nil {
t.Fatalf("test %d, item %d, error: %v", i, count, err)
}
// First item in iterator should have the expected key
if count == 0 && k != tc.expkey {
t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey)
}
count++
if v != k {
t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k)
}
}
}
}

Some files were not shown because too many files have changed in this diff Show more