forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/release/1.9' into mempool-feed

Browse source
This commit is contained in:
Austin Roberts 2020-04-16 09:14:55 -05:00
commit 24eb6797e2
242 changed files with 14431 additions and 2747 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE.md vendored
View file

@ -2,7 +2,7 @@ Hi there,
Please note that this is an issue tracker reserved for bug reports and feature requests.
For general questions please use the gitter channel or the Ethereum stack exchange at https://ethereum.stackexchange.com.
For general questions please use [discord](https://discord.gg/nthXNEv) or the Ethereum stack exchange at https://ethereum.stackexchange.com.
#### System information

55
.travis.yml
View file

@ -2,12 +2,21 @@ language: go
go_import_path: github.com/ethereum/go-ethereum
sudo: false
jobs:
allow_failures:
- stage: build
os: osx
go: 1.14.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
include:
# This builder only tests code linters on latest version of Go
# This builder only tests code linters on latest version of Go
- stage: lint
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- lint
git:
@ -35,12 +44,24 @@ jobs:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
os: linux
dist: xenial
go: 1.13.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
# These are the latest Go versions.
- stage: build
os: linux
arch: amd64
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
@ -50,7 +71,9 @@ jobs:
os: linux
arch: arm64
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
@ -58,7 +81,9 @@ jobs:
- stage: build
os: osx
osx_image: xcode11.3
go: 1.13.x
go: 1.14.x
env:
- GO111MODULE=on
script:
- echo "Increase the maximum number of open file descriptors on macOS"
- NOFILE=20480
@ -77,9 +102,10 @@ jobs:
if: type = push
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- ubuntu-ppa
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
@ -93,7 +119,7 @@ jobs:
- python-paramiko
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -goversion 1.13.8 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
- go run build/ci.go debsrc -goversion 1.14.2 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- stage: build
@ -101,9 +127,10 @@ jobs:
os: linux
dist: xenial
sudo: required
go: 1.13.x
go: 1.14.x
env:
- azure-linux
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
addons:
@ -137,9 +164,10 @@ jobs:
dist: xenial
services:
- docker
go: 1.13.x
go: 1.14.x
env:
- azure-linux-mips
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
@ -180,10 +208,11 @@ jobs:
env:
- azure-android
- maven-android
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- curl https://dl.google.com/go/go1.13.8.linux-amd64.tar.gz | tar -xz
- curl https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
@ -201,11 +230,12 @@ jobs:
- stage: build
if: type = push
os: osx
go: 1.13.x
go: 1.14.x
env:
- azure-osx
- azure-ios
- cocoapods-ios
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:
@ -232,9 +262,10 @@ jobs:
if: type = cron
os: linux
dist: xenial
go: 1.13.x
go: 1.14.x
env:
- azure-purge
- GO111MODULE=on
git:
submodules: false # avoid cloning ethereum/tests
script:

2
Dockerfile
View file

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.13-alpine as builder
FROM golang:1.14-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers git

2
Dockerfile.alltools
View file

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container
FROM golang:1.13-alpine as builder
FROM golang:1.14-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers git

45
README.md
View file

@ -16,7 +16,7 @@ archives are published at https://geth.ethereum.org/downloads/.
For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki.
Building `geth` requires both a Go (version 1.10 or later) and a C compiler. You can install
Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install
them using your favourite package manager. Once the dependencies are installed, run
```shell
@ -72,7 +72,7 @@ This command will:
This tool is optional and if you leave it out you can always attach to an already running
`geth` instance with `geth attach`.
### A Full node on the Ethereum test network
### A Full node on the Görli test network
Transitioning towards developers, if you'd like to play around with creating Ethereum
contracts, you almost certainly would like to do that without any real money involved until
@ -81,23 +81,24 @@ network, you want to join the **test** network with your node, which is fully eq
the main network, but with play-Ether only.
```shell
$ geth --testnet console
$ geth --goerli console
```
The `console` subcommand has the exact same meaning as above and they are equally
useful on the testnet too. Please see above for their explanations if you've skipped here.
useful on the testnet too. Please, see above for their explanations if you've skipped here.
Specifying the `--testnet` flag, however, will reconfigure your `geth` instance a bit:
Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit:
* Instead of connecting the main Ethereum network, the client will connect to the Görli
test network, which uses different P2P bootnodes, different network IDs and genesis
states.
* Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth`
will nest itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on
will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on
Linux). Note, on OSX and Linux this also means that attaching to a running testnet node
requires the use of a custom endpoint since `geth attach` will try to attach to a
production node endpoint by default. E.g.
`geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by
production node endpoint by default, e.g.,
`geth attach <datadir>/goerli/geth.ipc`. Windows users are not affected by
this.
* Instead of connecting the main Ethereum network, the client will connect to the test
network, which uses different P2P bootnodes, different network IDs and genesis states.
*Note: Although there are some internal protective measures to prevent transactions from
crossing over between the main network and test network, you should make sure to always
@ -107,17 +108,26 @@ accounts available between them.*
### Full node on the Rinkeby test network
The above test network is a cross-client one based on the ethash proof-of-work consensus
algorithm. As such, it has certain extra overhead and is more susceptible to reorganization
attacks due to the network's low difficulty/security. Go Ethereum also supports connecting
to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io)
(operated by members of the community). This network is lighter, more secure, but is only
supported by go-ethereum.
Go Ethereum also supports connecting to the older proof-of-authority based test network
called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community.
```shell
$ geth --rinkeby console
```
### Full node on the Ropsten test network
In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The
Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such,
it has certain extra overhead and is more susceptible to reorganization attacks due to the
network's low difficulty/security.
```shell
$ geth --ropsten console
```
*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.*
### Configuration
As an alternative to passing the numerous flags to the `geth` binary, you can also pass a
@ -217,7 +227,8 @@ aware of and agree upon. This consists of a small JSON file (e.g. call it `genes
"eip158Block": 0,
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0
"petersburgBlock": 0,
"istanbulBlock": 0
},
"alloc": {},
"coinbase": "0x0000000000000000000000000000000000000000",

116
accounts/abi/abi.go
View file

@ -19,6 +19,7 @@ package abi
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
@ -32,6 +33,12 @@ type ABI struct {
Constructor Method
Methods map[string]Method
Events map[string]Event
// Additional "special" functions introduced in solidity v0.6.0.
// It's separated from the original default fallback. Each contract
// can only define one fallback and receive function.
Fallback Method // Note it's also used to represent legacy fallback before v0.6.0
Receive Method
}
// JSON returns a parsed ABI interface and error if it failed.
@ -42,7 +49,6 @@ func JSON(reader io.Reader) (ABI, error) {
if err := dec.Decode(&abi); err != nil {
return ABI{}, err
}
return abi, nil
}
@ -108,13 +114,22 @@ func (abi ABI) UnpackIntoMap(v map[string]interface{}, name string, data []byte)
// UnmarshalJSON implements json.Unmarshaler interface
func (abi *ABI) UnmarshalJSON(data []byte) error {
var fields []struct {
Type string
Name string
Constant bool
Type string
Name string
Inputs []Argument
Outputs []Argument
// Status indicator which can be: "pure", "view",
// "nonpayable" or "payable".
StateMutability string
Anonymous bool
Inputs []Argument
Outputs []Argument
// Deprecated Status indicators, but removed in v0.6.0.
Constant bool // True if function is either pure or view
Payable bool // True if function is payable
// Event relevant indicator represents the event is
// declared as anonymous.
Anonymous bool
}
if err := json.Unmarshal(data, &fields); err != nil {
return err
@ -126,22 +141,82 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
case "constructor":
abi.Constructor = Method{
Inputs: field.Inputs,
// Note for constructor the `StateMutability` can only
// be payable or nonpayable according to the output of
// compiler. So constant is always false.
StateMutability: field.StateMutability,
// Legacy fields, keep them for backward compatibility
Constant: field.Constant,
Payable: field.Payable,
}
// empty defaults to function according to the abi spec
case "function", "":
case "function":
name := field.Name
_, ok := abi.Methods[name]
for idx := 0; ok; idx++ {
name = fmt.Sprintf("%s%d", field.Name, idx)
_, ok = abi.Methods[name]
}
isConst := field.Constant || field.StateMutability == "pure" || field.StateMutability == "view"
abi.Methods[name] = Method{
Name: name,
RawName: field.Name,
Const: isConst,
Inputs: field.Inputs,
Outputs: field.Outputs,
Name: name,
RawName: field.Name,
StateMutability: field.StateMutability,
Inputs: field.Inputs,
Outputs: field.Outputs,
// Legacy fields, keep them for backward compatibility
Constant: field.Constant,
Payable: field.Payable,
}
case "fallback":
// New introduced function type in v0.6.0, check more detail
// here https://solidity.readthedocs.io/en/v0.6.0/contracts.html#fallback-function
if abi.HasFallback() {
return errors.New("only single fallback is allowed")
}
abi.Fallback = Method{
Name: "",
RawName: "",
// The `StateMutability` can only be payable or nonpayable,
// so the constant is always false.
StateMutability: field.StateMutability,
IsFallback: true,
// Fallback doesn't have any input or output
Inputs: nil,
Outputs: nil,
// Legacy fields, keep them for backward compatibility
Constant: field.Constant,
Payable: field.Payable,
}
case "receive":
// New introduced function type in v0.6.0, check more detail
// here https://solidity.readthedocs.io/en/v0.6.0/contracts.html#fallback-function
if abi.HasReceive() {
return errors.New("only single receive is allowed")
}
if field.StateMutability != "payable" {
return errors.New("the statemutability of receive can only be payable")
}
abi.Receive = Method{
Name: "",
RawName: "",
// The `StateMutability` can only be payable, so constant
// is always true while payable is always false.
StateMutability: field.StateMutability,
IsReceive: true,
// Receive doesn't have any input or output
Inputs: nil,
Outputs: nil,
// Legacy fields, keep them for backward compatibility
Constant: field.Constant,
Payable: field.Payable,
}
case "event":
name := field.Name
@ -158,7 +233,6 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
}
}
}
return nil
}
@ -186,3 +260,13 @@ func (abi *ABI) EventByID(topic common.Hash) (*Event, error) {
}
return nil, fmt.Errorf("no event with id: %#x", topic.Hex())
}
// HasFallback returns an indicator whether a fallback function is included.
func (abi *ABI) HasFallback() bool {
return abi.Fallback.IsFallback
}
// HasReceive returns an indicator whether a receive function is included.
func (abi *ABI) HasReceive() bool {
return abi.Receive.IsReceive
}

50
accounts/abi/abi_test.go
View file

@ -31,29 +31,29 @@ import (
const jsondata = `
[
{ "type" : "function", "name" : "balance", "constant" : true },
{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }
{ "type" : "function", "name" : "balance", "stateMutability" : "view" },
{ "type" : "function", "name" : "send", "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }
]`
const jsondata2 = `
[
{ "type" : "function", "name" : "balance", "constant" : true },
{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
{ "type" : "function", "name" : "test", "constant" : false, "inputs" : [ { "name" : "number", "type" : "uint32" } ] },
{ "type" : "function", "name" : "string", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "string" } ] },
{ "type" : "function", "name" : "bool", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "bool" } ] },
{ "type" : "function", "name" : "address", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address" } ] },
{ "type" : "function", "name" : "uint64[2]", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] },
{ "type" : "function", "name" : "uint64[]", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] },
{ "type" : "function", "name" : "foo", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] },
{ "type" : "function", "name" : "bar", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] },
{ "type" : "function", "name" : "slice", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
{ "type" : "function", "name" : "slice256", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] },
{ "type" : "function", "name" : "sliceAddress", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] },
{ "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray2", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] },
{ "type" : "function", "name" : "nestedSlice", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] }
{ "type" : "function", "name" : "balance", "stateMutability" : "view" },
{ "type" : "function", "name" : "send", "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
{ "type" : "function", "name" : "test", "inputs" : [ { "name" : "number", "type" : "uint32" } ] },
{ "type" : "function", "name" : "string", "inputs" : [ { "name" : "inputs", "type" : "string" } ] },
{ "type" : "function", "name" : "bool", "inputs" : [ { "name" : "inputs", "type" : "bool" } ] },
{ "type" : "function", "name" : "address", "inputs" : [ { "name" : "inputs", "type" : "address" } ] },
{ "type" : "function", "name" : "uint64[2]", "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] },
{ "type" : "function", "name" : "uint64[]", "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] },
{ "type" : "function", "name" : "foo", "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] },
{ "type" : "function", "name" : "bar", "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] },
{ "type" : "function", "name" : "slice", "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
{ "type" : "function", "name" : "slice256", "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] },
{ "type" : "function", "name" : "sliceAddress", "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] },
{ "type" : "function", "name" : "sliceMultiAddress", "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray", "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] },
{ "type" : "function", "name" : "nestedArray2", "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] },
{ "type" : "function", "name" : "nestedSlice", "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] }
]`
func TestReader(t *testing.T) {
@ -61,10 +61,10 @@ func TestReader(t *testing.T) {
exp := ABI{
Methods: map[string]Method{
"balance": {
"balance", "balance", true, nil, nil,
"balance", "balance", "view", false, false, false, false, nil, nil,
},
"send": {
"send", "send", false, []Argument{
"send", "send", "", false, false, false, false, []Argument{
{"amount", Uint256, false},
}, nil,
},
@ -173,7 +173,7 @@ func TestTestSlice(t *testing.T) {
func TestMethodSignature(t *testing.T) {
String, _ := NewType("string", "", nil)
m := Method{"foo", "foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
m := Method{"foo", "foo", "", false, false, false, false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
exp := "foo(string,string)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
@ -185,7 +185,7 @@ func TestMethodSignature(t *testing.T) {
}
uintt, _ := NewType("uint256", "", nil)
m = Method{"foo", "foo", false, []Argument{{"bar", uintt, false}}, nil}
m = Method{"foo", "foo", "", false, false, false, false, []Argument{{"bar", uintt, false}}, nil}
exp = "foo(uint256)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
@ -204,7 +204,7 @@ func TestMethodSignature(t *testing.T) {
{Name: "y", Type: "int256"},
}},
})
m = Method{"foo", "foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
m = Method{"foo", "foo", "", false, false, false, false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
exp = "foo((int256,int256[],(int256,int256)[],(int256,int256)[2]),string)"
if m.Sig() != exp {
t.Error("signature mismatch", exp, "!=", m.Sig())
@ -582,7 +582,7 @@ func TestInputFixedArrayAndVariableInputLength(t *testing.T) {
}
func TestDefaultFunctionParsing(t *testing.T) {
const definition = `[{ "name" : "balance" }]`
const definition = `[{ "name" : "balance", "type" : "function" }]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {

2
accounts/abi/argument.go
View file

@ -292,7 +292,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
retval := make([]interface{}, 0, arguments.LengthNonIndexed())
virtualArgs := 0
for index, arg := range arguments.NonIndexed() {
marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
marshalledValue, err := ToGoType((index+virtualArgs)*32, arg.Type, data)
if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) {
// If we have a static array, like [3]uint256, these are coded as
// just like uint256,uint256,uint256.

6
accounts/abi/bind/backends/simulated.go
View file

@ -124,7 +124,7 @@ func (b *SimulatedBackend) rollback() {
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
}
// stateByBlockNumber retrieves a state by a given blocknumber.
@ -480,7 +480,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
return nil
}
@ -593,7 +593,7 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database(), nil)
return nil
}

12
accounts/abi/bind/base.go
View file

@ -171,12 +171,24 @@ func (c *BoundContract) Transact(opts *TransactOpts, method string, params ...in
if err != nil {
return nil, err
}
// todo(rjl493456442) check the method is payable or not,
// reject invalid transaction at the first place
return c.transact(opts, &c.address, input)
}
// RawTransact initiates a transaction with the given raw calldata as the input.
// It's usually used to initiates transaction for invoking **Fallback** function.
func (c *BoundContract) RawTransact(opts *TransactOpts, calldata []byte) (*types.Transaction, error) {
// todo(rjl493456442) check the method is payable or not,
// reject invalid transaction at the first place
return c.transact(opts, &c.address, calldata)
}
// Transfer initiates a plain transaction to move funds to the contract, calling
// its default method if one is available.
func (c *BoundContract) Transfer(opts *TransactOpts) (*types.Transaction, error) {
// todo(rjl493456442) check the payable fallback or receive is defined
// or not, reject invalid transaction at the first place
return c.transact(opts, &c.address, nil)
}

35
accounts/abi/bind/bind.go
View file

@ -77,6 +77,8 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
calls = make(map[string]*tmplMethod)
transacts = make(map[string]*tmplMethod)
events = make(map[string]*tmplEvent)
fallback *tmplMethod
receive *tmplMethod
// identifiers are used to detect duplicated identifier of function
// and event. For all calls, transacts and events, abigen will generate
@ -92,7 +94,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
normalizedName := methodNormalizer[lang](alias(aliases, original.Name))
// Ensure there is no duplicated identifier
var identifiers = callIdentifiers
if !original.Const {
if !original.IsConstant() {
identifiers = transactIdentifiers
}
if identifiers[normalizedName] {
@ -121,7 +123,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
}
}
// Append the methods to the call or transact lists
if original.Const {
if original.IsConstant() {
calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
} else {
transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
@ -156,7 +158,13 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
// Append the event to the accumulator list
events[original.Name] = &tmplEvent{Original: original, Normalized: normalized}
}
// Add two special fallback functions if they exist
if evmABI.HasFallback() {
fallback = &tmplMethod{Original: evmABI.Fallback}
}
if evmABI.HasReceive() {
receive = &tmplMethod{Original: evmABI.Receive}
}
// There is no easy way to pass arbitrary java objects to the Go side.
if len(structs) > 0 && lang == LangJava {
return "", errors.New("java binding for tuple arguments is not supported yet")
@ -169,6 +177,8 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
Constructor: evmABI.Constructor,
Calls: calls,
Transacts: transacts,
Fallback: fallback,
Receive: receive,
Events: events,
Libraries: make(map[string]string),
}
@ -619,11 +629,22 @@ func formatMethod(method abi.Method, structs map[string]*tmplStruct) string {
outputs[i] += fmt.Sprintf(" %v", output.Name)
}
}
constant := ""
if method.Const {
constant = "constant "
// Extract meaningful state mutability of solidity method.
// If it's default value, never print it.
state := method.StateMutability
if state == "nonpayable" {
state = ""
}
return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
if state != "" {
state = state + " "
}
identity := fmt.Sprintf("function %v", method.RawName)
if method.IsFallback {
identity = "fallback"
} else if method.IsReceive {
identity = "receive"
}
return fmt.Sprintf("%s(%v) %sreturns(%v)", identity, strings.Join(inputs, ", "), state, strings.Join(outputs, ", "))
}
// formatEvent transforms raw event representation into a user friendly one.

121
accounts/abi/bind/bind_test.go
View file

File diff suppressed because one or more lines are too long

66
accounts/abi/bind/template.go
View file

@ -35,6 +35,8 @@ type tmplContract struct {
Constructor abi.Method // Contract constructor for deploy parametrization
Calls map[string]*tmplMethod // Contract calls that only read state data
Transacts map[string]*tmplMethod // Contract calls that write state data
Fallback *tmplMethod // Additional special fallback function
Receive *tmplMethod // Additional special receive function
Events map[string]*tmplEvent // Contract events accessors
Libraries map[string]string // Same as tmplData, but filtered to only keep what the contract needs
Library bool // Indicator whether the contract is a library
@ -351,6 +353,52 @@ var (
}
{{end}}
{{if .Fallback}}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Transactor) Fallback(opts *bind.TransactOpts, calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.contract.RawTransact(opts, calldata)
}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Session) Fallback(calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Fallback(&_{{$contract.Type}}.TransactOpts, calldata)
}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) Fallback(calldata []byte) (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Fallback(&_{{$contract.Type}}.TransactOpts, calldata)
}
{{end}}
{{if .Receive}}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Transactor) Receive(opts *bind.TransactOpts) (*types.Transaction, error) {
return _{{$contract.Type}}.contract.RawTransact(opts, nil) // calldata is disallowed for receive function
}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}Session) Receive() (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Receive(&_{{$contract.Type}}.TransactOpts)
}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) Receive() (*types.Transaction, error) {
return _{{$contract.Type}}.Contract.Receive(&_{{$contract.Type}}.TransactOpts)
}
{{end}}
{{range .Events}}
// {{$contract.Type}}{{.Normalized.Name}}Iterator is returned from Filter{{.Normalized.Name}} and is used to iterate over the raw logs and unpacked data for {{.Normalized.Name}} events raised by the {{$contract.Type}} contract.
type {{$contract.Type}}{{.Normalized.Name}}Iterator struct {
@ -611,6 +659,24 @@ import java.util.*;
return this.Contract.transact(opts, "{{.Original.Name}}" , args);
}
{{end}}
{{if .Fallback}}
// Fallback is a paid mutator transaction binding the contract fallback function.
//
// Solidity: {{formatmethod .Fallback.Original $structs}}
public Transaction Fallback(TransactOpts opts, byte[] calldata) throws Exception {
return this.Contract.rawTransact(opts, calldata);
}
{{end}}
{{if .Receive}}
// Receive is a paid mutator transaction binding the contract receive function.
//
// Solidity: {{formatmethod .Receive.Original $structs}}
public Transaction Receive(TransactOpts opts) throws Exception {
return this.Contract.rawTransact(opts, null);
}
{{end}}
}
{{end}}
`

180
accounts/abi/bind/topics.go
View file

@ -49,17 +49,13 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
topic[common.HashLength-1] = 1
}
case int8:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 1))
case int16:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 2))
case int32:
blob := big.NewInt(int64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(int64(rule), 4))
case int64:
blob := big.NewInt(rule).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
copy(topic[:], genIntType(rule, 8))
case uint8:
blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
copy(topic[common.HashLength-len(blob):], blob)
@ -103,151 +99,75 @@ func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
return topics, nil
}
// Big batch of reflect types for topic reconstruction.
var (
reflectHash = reflect.TypeOf(common.Hash{})
reflectAddress = reflect.TypeOf(common.Address{})
reflectBigInt = reflect.TypeOf(new(big.Int))
)
func genIntType(rule int64, size uint) []byte {
var topic [common.HashLength]byte
if rule < 0 {
// if a rule is negative, we need to put it into two's complement.
// extended to common.Hashlength bytes.
topic = [common.HashLength]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}
}
for i := uint(0); i < size; i++ {
topic[common.HashLength-i-1] = byte(rule >> (i * 8))
}
return topic[:]
}
// parseTopics converts the indexed topic fields into actual log field values.
//
// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
// hashes as the topic value!
func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) error {
// Sanity check that the fields and topics match up
if len(fields) != len(topics) {
return errors.New("topic/field count mismatch")
}
// Iterate over all the fields and reconstruct them from topics
for _, arg := range fields {
if !arg.Indexed {
return errors.New("non-indexed field in topic reconstruction")
}
field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
// Try to parse the topic back into the fields based on primitive types
switch field.Kind() {
case reflect.Bool:
if topics[0][common.HashLength-1] == 1 {
field.Set(reflect.ValueOf(true))
}
case reflect.Int8:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int8(num.Int64())))
case reflect.Int16:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int16(num.Int64())))
case reflect.Int32:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(int32(num.Int64())))
case reflect.Int64:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(num.Int64()))
case reflect.Uint8:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint8(num.Uint64())))
case reflect.Uint16:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint16(num.Uint64())))
case reflect.Uint32:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(uint32(num.Uint64())))
case reflect.Uint64:
num := new(big.Int).SetBytes(topics[0][:])
field.Set(reflect.ValueOf(num.Uint64()))
default:
// Ran out of plain primitive types, try custom types
switch field.Type() {
case reflectHash: // Also covers all dynamic types
field.Set(reflect.ValueOf(topics[0]))
case reflectAddress:
var addr common.Address
copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
field.Set(reflect.ValueOf(addr))
case reflectBigInt:
num := new(big.Int).SetBytes(topics[0][:])
if arg.Type.T == abi.IntTy {
if num.Cmp(abi.MaxInt256) > 0 {
num.Add(abi.MaxUint256, big.NewInt(0).Neg(num))
num.Add(num, big.NewInt(1))
num.Neg(num)
}
}
field.Set(reflect.ValueOf(num))
default:
// Ran out of custom types, try the crazies
switch {
// static byte array
case arg.Type.T == abi.FixedBytesTy:
reflect.Copy(field, reflect.ValueOf(topics[0][:arg.Type.Size]))
default:
return fmt.Errorf("unsupported indexed type: %v", arg.Type)
}
}
}
topics = topics[1:]
}
return nil
return parseTopicWithSetter(fields, topics,
func(arg abi.Argument, reconstr interface{}) {
field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
field.Set(reflect.ValueOf(reconstr))
})
}
// parseTopicsIntoMap converts the indexed topic field-value pairs into map key-value pairs
func parseTopicsIntoMap(out map[string]interface{}, fields abi.Arguments, topics []common.Hash) error {
return parseTopicWithSetter(fields, topics,
func(arg abi.Argument, reconstr interface{}) {
out[arg.Name] = reconstr
})
}
// parseTopicWithSetter converts the indexed topic field-value pairs and stores them using the
// provided set function.
//
// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
// hashes as the topic value!
func parseTopicWithSetter(fields abi.Arguments, topics []common.Hash, setter func(abi.Argument, interface{})) error {
// Sanity check that the fields and topics match up
if len(fields) != len(topics) {
return errors.New("topic/field count mismatch")
}
// Iterate over all the fields and reconstruct them from topics
for _, arg := range fields {
for i, arg := range fields {
if !arg.Indexed {
return errors.New("non-indexed field in topic reconstruction")
}
var reconstr interface{}
switch arg.Type.T {
case abi.BoolTy:
out[arg.Name] = topics[0][common.HashLength-1] == 1
case abi.IntTy, abi.UintTy:
out[arg.Name] = abi.ReadInteger(arg.Type.T, arg.Type.Kind, topics[0].Bytes())
case abi.AddressTy:
var addr common.Address
copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
out[arg.Name] = addr
case abi.HashTy:
out[arg.Name] = topics[0]
case abi.FixedBytesTy:
array, err := abi.ReadFixedBytes(arg.Type, topics[0].Bytes())
if err != nil {
return err
}
out[arg.Name] = array
case abi.TupleTy:
return errors.New("tuple type in topic reconstruction")
case abi.StringTy, abi.BytesTy, abi.SliceTy, abi.ArrayTy:
// Array types (including strings and bytes) have their keccak256 hashes stored in the topic- not a hash
// whose bytes can be decoded to the actual value- so the best we can do is retrieve that hash
out[arg.Name] = topics[0]
reconstr = topics[i]
case abi.FunctionTy:
if garbage := binary.BigEndian.Uint64(topics[0][0:8]); garbage != 0 {
return fmt.Errorf("bind: got improperly encoded function type, got %v", topics[0].Bytes())
if garbage := binary.BigEndian.Uint64(topics[i][0:8]); garbage != 0 {
return fmt.Errorf("bind: got improperly encoded function type, got %v", topics[i].Bytes())
}
var tmp [24]byte
copy(tmp[:], topics[0][8:32])
out[arg.Name] = tmp
default: // Not handling tuples
return fmt.Errorf("unsupported indexed type: %v", arg.Type)
copy(tmp[:], topics[i][8:32])
reconstr = tmp
default:
var err error
reconstr, err = abi.ToGoType(0, arg.Type, topics[i].Bytes())
if err != nil {
return err
}
}
topics = topics[1:]
// Use the setter function to store the value
setter(arg, reconstr)
}
return nil

91
accounts/abi/bind/topics_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
)
func TestMakeTopics(t *testing.T) {
@ -41,6 +42,80 @@ func TestMakeTopics(t *testing.T) {
[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
false,
},
{
"support common hash types in topics",
args{[][]interface{}{{common.Hash{1, 2, 3, 4, 5}}}},
[][]common.Hash{{common.Hash{1, 2, 3, 4, 5}}},
false,
},
{
"support address types in topics",
args{[][]interface{}{{common.Address{1, 2, 3, 4, 5}}}},
[][]common.Hash{{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5}}},
false,
},
{
"support *big.Int types in topics",
args{[][]interface{}{{big.NewInt(1).Lsh(big.NewInt(2), 254)}}},
[][]common.Hash{{common.Hash{128}}},
false,
},
{
"support boolean types in topics",
args{[][]interface{}{
{true},
{false},
}},
[][]common.Hash{
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0}},
},
false,
},
{
"support int/uint(8/16/32/64) types in topics",
args{[][]interface{}{
{int8(-2)},
{int16(-3)},
{int32(-4)},
{int64(-5)},
{int8(1)},
{int16(256)},
{int32(65536)},
{int64(4294967296)},
{uint8(1)},
{uint16(256)},
{uint32(65536)},
{uint64(4294967296)},
}},
[][]common.Hash{
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 254}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 253}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 252}},
{common.Hash{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 251}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}},
{common.Hash{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0}},
},
false,
},
{
"support string types in topics",
args{[][]interface{}{{"hello world"}}},
[][]common.Hash{{crypto.Keccak256Hash([]byte("hello world"))}},
false,
},
{
"support byte slice types in topics",
args{[][]interface{}{{[]byte{1, 2, 3}}}},
[][]common.Hash{{crypto.Keccak256Hash([]byte{1, 2, 3})}},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -84,6 +159,7 @@ func setupTopicsTests() []topicTest {
bytesType, _ := abi.NewType("bytes5", "", nil)
int8Type, _ := abi.NewType("int8", "", nil)
int256Type, _ := abi.NewType("int256", "", nil)
tupleType, _ := abi.NewType("tuple(int256,int8)", "", nil)
tests := []topicTest{
{
@ -145,6 +221,21 @@ func setupTopicsTests() []topicTest {
},
wantErr: false,
},
{
name: "tuple(int256, int8)",
args: args{
createObj: func() interface{} { return nil },
resultObj: func() interface{} { return nil },
resultMap: func() map[string]interface{} { return make(map[string]interface{}) },
fields: abi.Arguments{abi.Argument{
Name: "tupletype",
Type: tupleType,
Indexed: true,
}},
topics: []common.Hash{},
},
wantErr: true,
},
}
return tests

56
accounts/abi/method.go
View file

@ -41,10 +41,23 @@ type Method struct {
// * foo(uint,uint)
// The method name of the first one will be resolved as foo while the second one
// will be resolved as foo0.
Name string
// RawName is the raw method name parsed from ABI.
RawName string
Const bool
Name string
RawName string // RawName is the raw method name parsed from ABI
// StateMutability indicates the mutability state of method,
// the default value is nonpayable. It can be empty if the abi
// is generated by legacy compiler.
StateMutability string
// Legacy indicators generated by compiler before v0.6.0
Constant bool
Payable bool
// The following two flags indicates whether the method is a
// special fallback introduced in solidity v0.6.0
IsFallback bool
IsReceive bool
Inputs Arguments
Outputs Arguments
}
@ -57,6 +70,11 @@ type Method struct {
//
// Please note that "int" is substitute for its canonical representation "int256"
func (method Method) Sig() string {
// Short circuit if the method is special. Fallback
// and Receive don't have signature at all.
if method.IsFallback || method.IsReceive {
return ""
}
types := make([]string, len(method.Inputs))
for i, input := range method.Inputs {
types[i] = input.Type.String()
@ -76,11 +94,22 @@ func (method Method) String() string {
outputs[i] += fmt.Sprintf(" %v", output.Name)
}
}
constant := ""
if method.Const {
constant = "constant "
// Extract meaningful state mutability of solidity method.
// If it's default value, never print it.
state := method.StateMutability
if state == "nonpayable" {
state = ""
}
return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
if state != "" {
state = state + " "
}
identity := fmt.Sprintf("function %v", method.RawName)
if method.IsFallback {
identity = "fallback"
} else if method.IsReceive {
identity = "receive"
}
return fmt.Sprintf("%v(%v) %sreturns(%v)", identity, strings.Join(inputs, ", "), state, strings.Join(outputs, ", "))
}
// ID returns the canonical representation of the method's signature used by the
@ -88,3 +117,14 @@ func (method Method) String() string {
func (method Method) ID() []byte {
return crypto.Keccak256([]byte(method.Sig()))[:4]
}
// IsConstant returns the indicator whether the method is read-only.
func (method Method) IsConstant() bool {
return method.StateMutability == "view" || method.StateMutability == "pure" || method.Constant
}
// IsPayable returns the indicator whether the method can process
// plain ether transfers.
func (method Method) IsPayable() bool {
return method.StateMutability == "payable" || method.Payable
}

29
accounts/abi/method_test.go
View file

@ -23,13 +23,15 @@ import (
const methoddata = `
[
{"type": "function", "name": "balance", "constant": true },
{"type": "function", "name": "send", "constant": false, "inputs": [{ "name": "amount", "type": "uint256" }]},
{"type": "function", "name": "transfer", "constant": false, "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]},
{"type": "function", "name": "balance", "stateMutability": "view"},
{"type": "function", "name": "send", "inputs": [{ "name": "amount", "type": "uint256" }]},
{"type": "function", "name": "transfer", "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple"}],"name":"tuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[]"}],"name":"tupleSlice","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5]"}],"name":"tupleArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}
{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
{"stateMutability":"nonpayable","type":"fallback"},
{"stateMutability":"payable","type":"receive"}
]`
func TestMethodString(t *testing.T) {
@ -39,7 +41,7 @@ func TestMethodString(t *testing.T) {
}{
{
method: "balance",
expectation: "function balance() constant returns()",
expectation: "function balance() view returns()",
},
{
method: "send",
@ -65,6 +67,14 @@ func TestMethodString(t *testing.T) {
method: "complexTuple",
expectation: "function complexTuple((uint256,uint256)[5][] a) returns()",
},
{
method: "fallback",
expectation: "fallback() returns()",
},
{
method: "receive",
expectation: "receive() payable returns()",
},
}
abi, err := JSON(strings.NewReader(methoddata))
@ -73,7 +83,14 @@ func TestMethodString(t *testing.T) {
}
for _, test := range table {
got := abi.Methods[test.method].String()
var got string
if test.method == "fallback" {
got = abi.Fallback.String()
} else if test.method == "receive" {
got = abi.Receive.String()
} else {
got = abi.Methods[test.method].String()
}
if got != test.expectation {
t.Errorf("expected string to be %s, got %s", test.expectation, got)
}

30
accounts/abi/unpack.go
View file

@ -27,13 +27,9 @@ import (
var (
// MaxUint256 is the maximum value that can be represented by a uint256
MaxUint256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
big.NewInt(-1))
MaxUint256 = new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 256), common.Big1)
// MaxInt256 is the maximum value that can be represented by a int256
MaxInt256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
big.NewInt(-1))
MaxInt256 = new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 255), common.Big1)
)
// ReadInteger reads the integer based on its kind and returns the appropriate value
@ -56,17 +52,17 @@ func ReadInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
case reflect.Int64:
return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
default:
// the only case lefts for integer is int256/uint256.
// big.SetBytes can't tell if a number is negative, positive on itself.
// On EVM, if the returned number > max int256, it is negative.
// the only case left for integer is int256/uint256.
ret := new(big.Int).SetBytes(b)
if typ == UintTy {
return ret
}
if ret.Cmp(MaxInt256) > 0 {
ret.Add(MaxUint256, big.NewInt(0).Neg(ret))
ret.Add(ret, big.NewInt(1))
// big.SetBytes can't tell if a number is negative or positive in itself.
// On EVM, if the returned number > max int256, it is negative.
// A number is > max int256 if the bit at position 255 is set.
if ret.Bit(255) == 1 {
ret.Add(MaxUint256, new(big.Int).Neg(ret))
ret.Add(ret, common.Big1)
ret.Neg(ret)
}
return ret
@ -144,7 +140,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
elemSize := getTypeSize(*t.Elem)
for i, j := start, 0; j < size; i, j = i+elemSize, j+1 {
inter, err := toGoType(i, *t.Elem, output)
inter, err := ToGoType(i, *t.Elem, output)
if err != nil {
return nil, err
}
@ -161,7 +157,7 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
retval := reflect.New(t.Type).Elem()
virtualArgs := 0
for index, elem := range t.TupleElems {
marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output)
marshalledValue, err := ToGoType((index+virtualArgs)*32, *elem, output)
if elem.T == ArrayTy && !isDynamicType(*elem) {
// If we have a static array, like [3]uint256, these are coded as
// just like uint256,uint256,uint256.
@ -187,9 +183,9 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) {
return retval.Interface(), nil
}
// toGoType parses the output bytes and recursively assigns the value of these bytes
// ToGoType parses the output bytes and recursively assigns the value of these bytes
// into a go type with accordance with the ABI spec.
func toGoType(index int, t Type, output []byte) (interface{}, error) {
func ToGoType(index int, t Type, output []byte) (interface{}, error) {
if index+32 > len(output) {
return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32)
}

38
accounts/abi/unpack_test.go
View file

@ -443,7 +443,7 @@ var unpackTests = []unpackTest{
func TestUnpack(t *testing.T) {
for i, test := range unpackTests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
def := fmt.Sprintf(`[{ "name" : "method", "type": "function", "outputs": %s}]`, test.def)
abi, err := JSON(strings.NewReader(def))
if err != nil {
t.Fatalf("invalid ABI definition %s: %v", def, err)
@ -522,7 +522,7 @@ type methodMultiOutput struct {
func methodMultiReturn(require *require.Assertions) (ABI, []byte, methodMultiOutput) {
const definition = `[
{ "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
{ "name" : "multi", "type": "function", "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]`
var expected = methodMultiOutput{big.NewInt(1), "hello"}
abi, err := JSON(strings.NewReader(definition))
@ -611,7 +611,7 @@ func TestMethodMultiReturn(t *testing.T) {
}
func TestMultiReturnWithArray(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -634,7 +634,7 @@ func TestMultiReturnWithArray(t *testing.T) {
}
func TestMultiReturnWithStringArray(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -664,7 +664,7 @@ func TestMultiReturnWithStringArray(t *testing.T) {
}
func TestMultiReturnWithStringSlice(t *testing.T) {
const definition = `[{"name" : "multi", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -700,7 +700,7 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
// values of nested static arrays count towards the size as well, and any element following
// after such nested array argument should be read with the correct offset,
// so that it does not read content from the previous array argument.
const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]`
const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
t.Fatal(err)
@ -737,15 +737,15 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
func TestUnmarshal(t *testing.T) {
const definition = `[
{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },
{ "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] },
{ "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] },
{ "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] },
{ "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] },
{ "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] },
{ "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] },
{ "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] },
{ "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]`
{ "name" : "int", "type": "function", "outputs": [ { "type": "uint256" } ] },
{ "name" : "bool", "type": "function", "outputs": [ { "type": "bool" } ] },
{ "name" : "bytes", "type": "function", "outputs": [ { "type": "bytes" } ] },
{ "name" : "fixed", "type": "function", "outputs": [ { "type": "bytes32" } ] },
{ "name" : "multi", "type": "function", "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] },
{ "name" : "intArraySingle", "type": "function", "outputs": [ { "type": "uint256[3]" } ] },
{ "name" : "addressSliceSingle", "type": "function", "outputs": [ { "type": "address[]" } ] },
{ "name" : "addressSliceDouble", "type": "function", "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] },
{ "name" : "mixedBytes", "type": "function", "stateMutability" : "view", "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]`
abi, err := JSON(strings.NewReader(definition))
if err != nil {
@ -985,7 +985,7 @@ func TestUnmarshal(t *testing.T) {
}
func TestUnpackTuple(t *testing.T) {
const simpleTuple = `[{"name":"tuple","constant":false,"outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]`
const simpleTuple = `[{"name":"tuple","type":"function","outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]`
abi, err := JSON(strings.NewReader(simpleTuple))
if err != nil {
t.Fatal(err)
@ -1009,12 +1009,12 @@ func TestUnpackTuple(t *testing.T) {
t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.A)
}
if v.B.Cmp(big.NewInt(-1)) != 0 {
t.Errorf("unexpected value unpacked: want %x, got %x", v.B, -1)
t.Errorf("unexpected value unpacked: want %x, got %x", -1, v.B)
}
}
// Test nested tuple
const nestedTuple = `[{"name":"tuple","constant":false,"outputs":[
const nestedTuple = `[{"name":"tuple","type":"function","outputs":[
{"type":"tuple","name":"s","components":[{"type":"uint256","name":"a"},{"type":"uint256[]","name":"b"},{"type":"tuple[]","name":"c","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]}]},
{"type":"tuple","name":"t","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]},
{"type":"uint256","name":"a"}
@ -1136,7 +1136,7 @@ func TestOOMMaliciousInput(t *testing.T) {
},
}
for i, test := range oomTests {
def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
def := fmt.Sprintf(`[{ "name" : "method", "type": "function", "outputs": %s}]`, test.def)
abi, err := JSON(strings.NewReader(def))
if err != nil {
t.Fatalf("invalid ABI definition %s: %v", def, err)

6
accounts/scwallet/wallet.go
View file

@ -312,15 +312,15 @@ func (w *Wallet) Status() (string, error) {
}
switch {
case !w.session.verified && status.PinRetryCount == 0 && status.PukRetryCount == 0:
return fmt.Sprintf("Bricked, waiting for full wipe"), nil
return "Bricked, waiting for full wipe", nil
case !w.session.verified && status.PinRetryCount == 0:
return fmt.Sprintf("Blocked, waiting for PUK (%d attempts left) and new PIN", status.PukRetryCount), nil
case !w.session.verified:
return fmt.Sprintf("Locked, waiting for PIN (%d attempts left)", status.PinRetryCount), nil
case !status.Initialized:
return fmt.Sprintf("Empty, waiting for initialization"), nil
return "Empty, waiting for initialization", nil
default:
return fmt.Sprintf("Online"), nil
return "Online", nil
}
}

5
appveyor.yml
View file

@ -6,6 +6,7 @@ clone_depth: 5
version: "{branch}.{build}"
environment:
global:
GO111MODULE: on
GOPATH: C:\gopath
CC: gcc.exe
matrix:
@ -23,8 +24,8 @@ environment:
install:
- git submodule update --init
- rmdir C:\go /s /q
- appveyor DownloadFile https://dl.google.com/go/go1.13.8.windows-%GETH_ARCH%.zip
- 7z x go1.13.8.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- appveyor DownloadFile https://dl.google.com/go/go1.14.2.windows-%GETH_ARCH%.zip
- 7z x go1.14.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version

34
build/checksums.txt
View file

@ -1,19 +1,21 @@
# This file contains sha256 checksums of optional build dependencies.
b13bf04633d4d8cf53226ebeaace8d4d2fd07ae6fa676d0844a688339debec34 go1.13.8.src.tar.gz
98de84e69726a66da7b4e58eac41b99cbe274d7e8906eeb8a5b7eb0aadee7f7c go1.14.2.src.tar.gz
478994633b0f5121a7a8d4f368078093e21014fdc7fb2c0ceeae63668c13c5b6 golangci-lint-1.22.2-freebsd-amd64.tar.gz
fcf80824c21567eb0871055711bf9bdca91cf9a081122e2a45f1d11fed754600 golangci-lint-1.22.2-darwin-amd64.tar.gz
cda85c72fc128b2ea0ae05baea7b91172c63aea34064829f65285f1dd536f1e0 golangci-lint-1.22.2-windows-386.zip
94f04899f620aadc9c1524e5482e415efdbd993fa2b2918c4fec2798f030ac1c golangci-lint-1.22.2-linux-armv7.tar.gz
0e72a87d71edde00b6e37e84a99841833ad55fee83e20d21130a7a622b2860bb golangci-lint-1.22.2-freebsd-386.tar.gz
86def2f31fe8fd7c05674104ed2a4bef3e44b7132b93c6ad2f52f198b3d01801 golangci-lint-1.22.2-linux-s390x.tar.gz
b0df4546d36be94e8107733ba290b98dd9b7e41a42d3fb202e87fc7e4ee800c3 golangci-lint-1.22.2-freebsd-armv6.tar.gz
3d45958dcf6a8d195086d2fced1a21db42a90815dfd156d180efa62dbdda6724 golangci-lint-1.22.2-darwin-386.tar.gz
7ee29f35c74fab017a454237990c74d984ce3855960f2c10509238992bb781f9 golangci-lint-1.22.2-linux-arm64.tar.gz
52086ac52a502b68578e58e35d3964f127c16d7a90b9ffcb399a004d055ded51 golangci-lint-1.22.2-linux-386.tar.gz
c2e4df1fab2ae53762f9baac6041503eeeaa968ce38ea41779f7cb526751c667 golangci-lint-1.22.2-windows-amd64.zip
109d38cdc89f271392f5a138d6782657157f9f496fd4801956efa2d0428e0cbe golangci-lint-1.22.2-linux-amd64.tar.gz
f08aae4868d4828c8f07deb0dcd941a1da695b97e58d15e9f3d1d07dcc7a0c84 golangci-lint-1.22.2-linux-armv6.tar.gz
37af03d9c144d527cb15c46a07e6a22d3f62b5491e34ad6f3bfe6bb0b0b597d4 golangci-lint-1.22.2-linux-ppc64le.tar.gz
251a1081d53944f1d5f86216d752837b23079f90605c9d1cc628da1ffcd2e749 golangci-lint-1.22.2-freebsd-armv7.tar.gz
aeaa5498682246b87d0b77ece283897348ea03d98e816760a074058bfca60b2a golangci-lint-1.24.0-windows-amd64.zip
7e854a70d449fe77b7a91583ec88c8603eb3bf96c45d52797dc4ba3f2f278dbe golangci-lint-1.24.0-darwin-386.tar.gz
835101fae192c3a2e7a51cb19d5ac3e1a40b0e311955e89bc21d61de78635979 golangci-lint-1.24.0-linux-armv6.tar.gz
a041a6e6a61c9ff3dbe58673af13ea00c76bcd462abede0ade645808e97cdd6d golangci-lint-1.24.0-windows-386.zip
7cc73eb9ca02b7a766c72b913f8080401862b10e7bb90c09b085415a81f21609 golangci-lint-1.24.0-freebsd-armv6.tar.gz
537bb2186987b5e68ad4e8829230557f26087c3028eb736dea1662a851bad73d golangci-lint-1.24.0-linux-armv7.tar.gz
8cb1bc1e63d8f0d9b71fcb10b38887e1646a6b8a120ded2e0cd7c3284528f633 golangci-lint-1.24.0-linux-mips64.tar.gz
095d3f8bf7fc431739861574d0b58d411a617df2ed5698ce5ae5ecc66d23d44d golangci-lint-1.24.0-freebsd-armv7.tar.gz
e245df27cec3827aef9e7afbac59e92816978ee3b64f84f7b88562ff4b2ac225 golangci-lint-1.24.0-linux-arm64.tar.gz
35d6d5927e19f0577cf527f0e4441dbb37701d87e8cf729c98a510fce397fbf7 golangci-lint-1.24.0-linux-ppc64le.tar.gz
a1ed66353b8ceb575d78db3051491bce3ac1560e469a9bc87e8554486fec7dfe golangci-lint-1.24.0-freebsd-386.tar.gz
241ca454102e909de04957ff8a5754c757cefa255758b3e1fba8a4533d19d179 golangci-lint-1.24.0-linux-amd64.tar.gz
ff488423db01a0ec8ffbe4e1d65ef1be6a8d5e6d7930cf380ce8aaf714125470 golangci-lint-1.24.0-linux-386.tar.gz
f05af56f15ebbcf77663a8955d1e39009b584ce8ea4c5583669369d80353a113 golangci-lint-1.24.0-darwin-amd64.tar.gz
b0096796c0ffcd6c350a2ec006100e7ef5f0597b43a204349d4f997273fb32a7 golangci-lint-1.24.0-freebsd-amd64.tar.gz
c9c2867380e85628813f1f7d1c3cfc6c6f7931e89bea86f567ff451b8cdb6654 golangci-lint-1.24.0-linux-mips64le.tar.gz
2feb97fa61c934aa3eba9bc104ab5dd8fb946791d58e64060e8857e800eeae0b golangci-lint-1.24.0-linux-s390x.tar.gz

2
build/ci.go
View file

@ -356,7 +356,7 @@ func doLint(cmdline []string) {
// downloadLinter downloads and unpacks golangci-lint.
func downloadLinter(cachedir string) string {
const version = "1.22.2"
const version = "1.24.0"
csdb := build.MustLoadChecksums("build/checksums.txt")
base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)

103
cmd/checkpoint-admin/README.md Normal file
View file

@ -0,0 +1,103 @@
## Checkpoint-admin
Checkpoint-admin is a tool for updating checkpoint oracle status. It provides a series of functions including deploying checkpoint oracle contract, signing for new checkpoints, and updating checkpoints in the checkpoint oracle contract.
### Checkpoint
In the LES protocol, there is an important concept called checkpoint. In simple terms, whenever a certain number of blocks are generated on the blockchain, a new checkpoint is generated which contains some important information such as
* Block hash at checkpoint
* Canonical hash trie root at checkpoint
* Bloom trie root at checkpoint
*For a more detailed introduction to checkpoint, please see the LES [spec](https://github.com/ethereum/devp2p/blob/master/caps/les.md).*
Using this information, light clients can skip all historical block headers when synchronizing data and start synchronization from this checkpoint. Therefore, as long as the light client can obtain some latest and correct checkpoints, the amount of data and time for synchronization will be greatly reduced.
However, from a security perspective, the most critical step in a synchronization algorithm based on checkpoints is to determine whether the checkpoint used by the light client is correct. Otherwise, all blockchain data synchronized based on this checkpoint may be wrong. For this we provide two different ways to ensure the correctness of the checkpoint used by the light client.
#### Hardcoded checkpoint
There are several hardcoded checkpoints in the [source code](https://github.com/ethereum/go-ethereum/blob/master/params/config.go#L38) of the go-ethereum project. These checkpoints are updated by go-ethereum developers when new versions of software are released. Because light client users trust Geth developers to some extent, hardcoded checkpoints in the code can also be considered correct.
#### Checkpoint oracle
Hardcoded checkpoints can solve the problem of verifying the correctness of checkpoints (although this is a more centralized solution). But the pain point of this solution is that developers can only update checkpoints when a new version of software is released. In addition, light client users usually do not keep the Geth version they use always up to date. So hardcoded checkpoints used by users are generally stale. Therefore, it still needs to download a large amount of blockchain data during synchronization.
Checkpoint oracle is a more flexible solution. In simple terms, this is a smart contract that is deployed on the blockchain. The smart contract records several designated trusted signers. Whenever enough trusted signers have issued their signatures for the same checkpoint, it can be considered that the checkpoint has been authenticated by the signers. Checkpoints authenticated by trusted signers can be considered correct.
So this way, even without updating the software version, as long as the trusted signers regularly update the checkpoint in oracle on time, the light client can always use the latest and verified checkpoint for data synchronization.
### Usage
Checkpoint-admin is a command line tool designed for checkpoint oracle. Users can easily deploy contracts and update checkpoints through this tool.
#### Install
```shell
go get github.com/ethereum/go-ethereum/cmd/checkpoint-admin
```
#### Deploy
Deploy checkpoint oracle contract. `--signers` indicates the specified trusted signer, and `--threshold` indicates the minimum number of signatures required by trusted signers to update a checkpoint.
```shell
checkpoint-admin deploy --rpc <NODE_RPC_ENDPOINT> --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_TX> --signers <TRUSTED_SIGNER_LIST> --threshold 1
```
It is worth noting that checkpoint-admin only supports clef as a signer for transactions and plain text(checkpoint). For more clef usage, please see the clef [tutorial](https://geth.ethereum.org/docs/clef/tutorial) .
#### Sign
Checkpoint-admin provides two different modes of signing. You can automatically obtain the current stable checkpoint and sign it interactively, and you can also use the information provided by the command line flags to sign checkpoint offline.
**Interactive mode**
```shell
checkpoint-admin sign --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_CHECKPOINT> --rpc <NODE_RPC_ENDPOINT>
```
*It is worth noting that the connected Geth node can be a fullnode or a light client. If it is fullnode, you must enable the LES protocol. E.G. add `--light.serv 50` to the startup command line flags*.
**Offline mode**
```shell
checkpoint-admin sign --clef <CLEF_ENDPOINT> --signer <SIGNER_TO_SIGN_CHECKPOINT> --index <CHECKPOINT_INDEX> --hash <CHECKPOINT_HASH> --oracle <CHECKPOINT_ORACLE_ADDRESS>
```
*CHECKPOINT_HASH is obtained based on this [calculation method](https://github.com/ethereum/go-ethereum/blob/master/params/config.go#L251).*
#### Publish
Collect enough signatures from different trusted signers for the same checkpoint and submit them to oracle to update the "authenticated" checkpoint in the contract.
```shell
checkpoint-admin publish --clef <CLEF_ENDPOINT> --rpc <NODE_RPC_ENDPOINT> --signer <SIGNER_TO_SIGN_TX> --index <CHECKPOINT_INDEX> --signatures <CHECKPOINT_SIGNATURE_LIST>
```
#### Status query
Check the latest status of checkpoint oracle.
```shell
checkpoint-admin status --rpc <NODE_RPC_ENDPOINT>
```
### Enable checkpoint oracle in your private network
Currently, only the Ethereum mainnet and the default supported test networks (ropsten, rinkeby, goerli) activate this feature. If you want to activate this feature in your private network, you can overwrite the relevant checkpoint oracle settings through the configuration file after deploying the oracle contract.
* Get your node configuration file `geth dumpconfig OTHER_COMMAND_LINE_OPTIONS > config.toml`
* Edit the configuration file and add the following information
```toml
[Eth.CheckpointOracle]
Address = CHECKPOINT_ORACLE_ADDRESS
Signers = [TRUSTED_SIGNER_1, ..., TRUSTED_SIGNER_N]
Threshold = THRESHOLD
```
* Start geth with the modified configuration file
*In the private network, all fullnodes and light clients need to be started using the same checkpoint oracle settings.*

14
cmd/clef/docs/setup.md
View file

@ -34,7 +34,7 @@ There are two ways that this can be achieved: integrated via Qubes or integrated
#### 1. Qubes Integrated
Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request
Qubes provides a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request
to another qube. The OS then asks the user if the call is permitted.
![Example](qubes/qrexec-example.png)
@ -48,7 +48,7 @@ This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented.
![Clef via qrexec](qubes/clef_qubes_qrexec.png)
On the `target` qubes, we need to define the rpc service.
On the `target` qubes, we need to define the RPC service.
[qubes.Clefsign](qubes/qubes.Clefsign):
@ -135,11 +135,11 @@ $ cat newaccnt.json
$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign
```
This should pop up first a dialog to allow the IPC call:
A dialog should pop up first to allow the IPC call:
![one](qubes/qubes_newaccount-1.png)
Followed by a GTK-dialog to approve the operation
Followed by a GTK-dialog to approve the operation:
![two](qubes/qubes_newaccount-2.png)
@ -169,7 +169,7 @@ However, it comes with a couple of drawbacks:
- The `Origin` header must be forwarded
- Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header,
since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address.
- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups:
- Even with a policy in place to allow RPC calls between `caller` and `target`, there will be several popups:
- One qubes-specific where the user specifies the `target` vm
- One clef-specific to approve the transaction
@ -177,7 +177,7 @@ However, it comes with a couple of drawbacks:
#### 2. Network integrated
The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible
form other qubes.
from other qubes.
![Clef via http](qubes/clef_qubes_http.png)
@ -193,6 +193,6 @@ to your computer. Over this new network interface, you can SSH into the device.
Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only
ever connects to a local network between your computer and the device itself.
Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access
Needless to say, while this model should be fairly secure against remote attacks, an attacker with physical access
to the USB Armory would trivially be able to extract the contents of the device filesystem.

11
cmd/clef/intapi_changelog.md
View file

@ -10,6 +10,17 @@ TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
### 7.0.1
Added `clef_New` to the internal API calleable from a UI.
> `New` creates a new password protected Account. The private key is protected with
> the given password. Users are responsible to backup the private key that is stored
> in the keystore location that was specified when this API was created.
> This method is the same as New on the external API, the difference being that
> this implementation does not ask for confirmation, since it's initiated by
> the user
### 7.0.0
- The `message` field was renamed to `messages` in all data signing request methods to better reflect that it's a list, not a value.

61
cmd/clef/main.go
View file

@ -187,6 +187,21 @@ The setpw command stores a password for a given address (keyfile).
Description: `
The delpw command removes a password for a given address (keyfile).
`}
newAccountCommand = cli.Command{
Action: utils.MigrateFlags(newAccount),
Name: "newaccount",
Usage: "Create a new account",
ArgsUsage: "",
Flags: []cli.Flag{
logLevelFlag,
keystoreFlag,
utils.LightKDFFlag,
},
Description: `
The newaccount command creates a new keystore-backed account. It is a convenience-method
which can be used in lieu of an external UI.`,
}
gendocCommand = cli.Command{
Action: GenDoc,
Name: "gendoc",
@ -222,7 +237,12 @@ func init() {
advancedMode,
}
app.Action = signer
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, gendocCommand}
app.Commands = []cli.Command{initCommand,
attestCommand,
setCredentialCommand,
delCredentialCommand,
newAccountCommand,
gendocCommand}
cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate
}
@ -382,6 +402,31 @@ func removeCredential(ctx *cli.Context) error {
return nil
}
func newAccount(c *cli.Context) error {
if err := initialize(c); err != nil {
return err
}
// The newaccount is meant for users using the CLI, since 'real' external
// UIs can use the UI-api instead. So we'll just use the native CLI UI here.
var (
ui = core.NewCommandlineUI()
pwStorage storage.Storage = &storage.NoStorage{}
ksLoc = c.GlobalString(keystoreFlag.Name)
lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
)
log.Info("Starting clef", "keystore", ksLoc, "light-kdf", lightKdf)
am := core.StartClefAccountManager(ksLoc, true, lightKdf, "")
// This gives is us access to the external API
apiImpl := core.NewSignerAPI(am, 0, true, ui, nil, false, pwStorage)
// This gives us access to the internal API
internalApi := core.NewUIServerAPI(apiImpl)
addr, err := internalApi.New(context.Background())
if err == nil {
fmt.Printf("Generated account %v\n", addr.String())
}
return err
}
func initialize(c *cli.Context) error {
// Set up the logger to print everything
logOutput := os.Stdout
@ -457,7 +502,6 @@ func signer(c *cli.Context) error {
api core.ExternalAPI
pwStorage storage.Storage = &storage.NoStorage{}
)
configDir := c.GlobalString(configdirFlag.Name)
if stretchedKey, err := readMasterKey(c, ui); err != nil {
log.Warn("Failed to open master, rules disabled", "err", err)
@ -539,18 +583,25 @@ func signer(c *cli.Context) error {
vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
srv := rpc.NewServer()
err := node.RegisterApisFromWhitelist(rpcAPI, []string{"account"}, srv, false)
if err != nil {
utils.Fatalf("Could not register API: %w", err)
}
handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
// start http server
httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
listener, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler)
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}
extapiURL = fmt.Sprintf("http://%s", httpEndpoint)
extapiURL = fmt.Sprintf("http://%v/", listener.Addr())
log.Info("HTTP endpoint opened", "url", extapiURL)
defer func() {
listener.Close()
log.Info("HTTP endpoint closed", "url", httpEndpoint)
log.Info("HTTP endpoint closed", "url", extapiURL)
}()
}
if !c.GlobalBool(utils.IPCDisabledFlag.Name) {

10
cmd/devp2p/crawl.go
View file

@ -20,14 +20,13 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/enode"
)
type crawler struct {
input nodeSet
output nodeSet
disc *discover.UDPv4
disc resolver
iters []enode.Iterator
inputIter enode.Iterator
ch chan *enode.Node
@ -37,7 +36,11 @@ type crawler struct {
revalidateInterval time.Duration
}
func newCrawler(input nodeSet, disc *discover.UDPv4, iters ...enode.Iterator) *crawler {
type resolver interface {
RequestENR(*enode.Node) (*enode.Node, error)
}
func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler {
c := &crawler{
input: input,
output: make(nodeSet, len(input)),
@ -63,6 +66,7 @@ func (c *crawler) run(timeout time.Duration) nodeSet {
doneCh = make(chan enode.Iterator, len(c.iters))
liveIters = len(c.iters)
)
defer timeoutTimer.Stop()
for _, it := range c.iters {
go c.runIterator(doneCh, it)
}

105
cmd/devp2p/discv4cmd.go
View file

@ -81,6 +81,18 @@ var (
Name: "bootnodes",
Usage: "Comma separated nodes used for bootstrapping",
}
nodekeyFlag = cli.StringFlag{
Name: "nodekey",
Usage: "Hex-encoded node key",
}
nodedbFlag = cli.StringFlag{
Name: "nodedb",
Usage: "Nodes database location",
}
listenAddrFlag = cli.StringFlag{
Name: "addr",
Usage: "Listening address",
}
crawlTimeoutFlag = cli.DurationFlag{
Name: "timeout",
Usage: "Time limit for the crawl.",
@ -172,6 +184,62 @@ func discv4Crawl(ctx *cli.Context) error {
return nil
}
// startV4 starts an ephemeral discovery V4 node.
func startV4(ctx *cli.Context) *discover.UDPv4 {
ln, config := makeDiscoveryConfig(ctx)
socket := listen(ln, ctx.String(listenAddrFlag.Name))
disc, err := discover.ListenV4(socket, ln, config)
if err != nil {
exit(err)
}
return disc
}
func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) {
var cfg discover.Config
if ctx.IsSet(nodekeyFlag.Name) {
key, err := crypto.HexToECDSA(ctx.String(nodekeyFlag.Name))
if err != nil {
exit(fmt.Errorf("-%s: %v", nodekeyFlag.Name, err))
}
cfg.PrivateKey = key
} else {
cfg.PrivateKey, _ = crypto.GenerateKey()
}
if commandHasFlag(ctx, bootnodesFlag) {
bn, err := parseBootnodes(ctx)
if err != nil {
exit(err)
}
cfg.Bootnodes = bn
}
dbpath := ctx.String(nodedbFlag.Name)
db, err := enode.OpenDB(dbpath)
if err != nil {
exit(err)
}
ln := enode.NewLocalNode(db, cfg.PrivateKey)
return ln, cfg
}
func listen(ln *enode.LocalNode, addr string) *net.UDPConn {
if addr == "" {
addr = "0.0.0.0:0"
}
socket, err := net.ListenPacket("udp4", addr)
if err != nil {
exit(err)
}
usocket := socket.(*net.UDPConn)
uaddr := socket.LocalAddr().(*net.UDPAddr)
ln.SetFallbackIP(net.IP{127, 0, 0, 1})
ln.SetFallbackUDP(uaddr.Port)
return usocket
}
func parseBootnodes(ctx *cli.Context) ([]*enode.Node, error) {
s := params.RinkebyBootnodes
if ctx.IsSet(bootnodesFlag.Name) {
@ -187,40 +255,3 @@ func parseBootnodes(ctx *cli.Context) ([]*enode.Node, error) {
}
return nodes, nil
}
// startV4 starts an ephemeral discovery V4 node.
func startV4(ctx *cli.Context) *discover.UDPv4 {
socket, ln, cfg, err := listen()
if err != nil {
exit(err)
}
if commandHasFlag(ctx, bootnodesFlag) {
bn, err := parseBootnodes(ctx)
if err != nil {
exit(err)
}
cfg.Bootnodes = bn
}
disc, err := discover.ListenV4(socket, ln, cfg)
if err != nil {
exit(err)
}
return disc
}
func listen() (*net.UDPConn, *enode.LocalNode, discover.Config, error) {
var cfg discover.Config
cfg.PrivateKey, _ = crypto.GenerateKey()
db, _ := enode.OpenDB("")
ln := enode.NewLocalNode(db, cfg.PrivateKey)
socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{0, 0, 0, 0}})
if err != nil {
db.Close()
return nil, nil, cfg, err
}
addr := socket.LocalAddr().(*net.UDPAddr)
ln.SetFallbackIP(net.IP{127, 0, 0, 1})
ln.SetFallbackUDP(addr.Port)
return socket, ln, cfg, nil
}

123
cmd/devp2p/discv5cmd.go Normal file
View file

@ -0,0 +1,123 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p/discover"
"gopkg.in/urfave/cli.v1"
)
var (
discv5Command = cli.Command{
Name: "discv5",
Usage: "Node Discovery v5 tools",
Subcommands: []cli.Command{
discv5PingCommand,
discv5ResolveCommand,
discv5CrawlCommand,
discv5ListenCommand,
},
}
discv5PingCommand = cli.Command{
Name: "ping",
Usage: "Sends ping to a node",
Action: discv5Ping,
}
discv5ResolveCommand = cli.Command{
Name: "resolve",
Usage: "Finds a node in the DHT",
Action: discv5Resolve,
Flags: []cli.Flag{bootnodesFlag},
}
discv5CrawlCommand = cli.Command{
Name: "crawl",
Usage: "Updates a nodes.json file with random nodes found in the DHT",
Action: discv5Crawl,
Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag},
}
discv5ListenCommand = cli.Command{
Name: "listen",
Usage: "Runs a node",
Action: discv5Listen,
Flags: []cli.Flag{
bootnodesFlag,
nodekeyFlag,
nodedbFlag,
listenAddrFlag,
},
}
)
func discv5Ping(ctx *cli.Context) error {
n := getNodeArg(ctx)
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Ping(n))
return nil
}
func discv5Resolve(ctx *cli.Context) error {
n := getNodeArg(ctx)
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Resolve(n))
return nil
}
func discv5Crawl(ctx *cli.Context) error {
if ctx.NArg() < 1 {
return fmt.Errorf("need nodes file as argument")
}
nodesFile := ctx.Args().First()
var inputSet nodeSet
if common.FileExist(nodesFile) {
inputSet = loadNodesJSON(nodesFile)
}
disc := startV5(ctx)
defer disc.Close()
c := newCrawler(inputSet, disc, disc.RandomNodes())
c.revalidateInterval = 10 * time.Minute
output := c.run(ctx.Duration(crawlTimeoutFlag.Name))
writeNodesJSON(nodesFile, output)
return nil
}
func discv5Listen(ctx *cli.Context) error {
disc := startV5(ctx)
defer disc.Close()
fmt.Println(disc.Self())
select {}
}
// startV5 starts an ephemeral discovery v5 node.
func startV5(ctx *cli.Context) *discover.UDPv5 {
ln, config := makeDiscoveryConfig(ctx)
socket := listen(ln, ctx.String(listenAddrFlag.Name))
disc, err := discover.ListenV5(socket, ln, config)
if err != nil {
exit(err)
}
return disc
}

39
cmd/devp2p/dns_route53.go
View file

@ -32,9 +32,13 @@ import (
"gopkg.in/urfave/cli.v1"
)
// The Route53 limits change sets to this size. DNS changes need to be split
// up into multiple batches to work around the limit.
const route53ChangeLimit = 30000
const (
// Route53 limits change sets to 32k of 'RDATA size'. Change sets are also limited to
// 1000 items. UPSERTs count double.
// https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets
route53ChangeSizeLimit = 32000
route53ChangeCountLimit = 1000
)
var (
route53AccessKeyFlag = cli.StringFlag{
@ -102,7 +106,7 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error {
}
// Submit change batches.
batches := splitChanges(changes, route53ChangeLimit)
batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit)
for i, changes := range batches {
log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes)))
batch := new(route53.ChangeBatch)
@ -176,7 +180,7 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e
// Entry is unknown, push a new one
log.Info(fmt.Sprintf("Creating %s = %q", path, val))
changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val)))
} else if prevValue != val {
} else if prevValue != val || prevRecords.ttl != ttl {
// Entry already exists, only change its content.
log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val))
changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val)))
@ -212,18 +216,26 @@ func sortChanges(changes []*route53.Change) {
// splitChanges splits up DNS changes such that each change batch
// is smaller than the given RDATA limit.
func splitChanges(changes []*route53.Change, limit int) [][]*route53.Change {
var batches [][]*route53.Change
var batchSize int
func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*route53.Change {
var (
batches [][]*route53.Change
batchSize int
batchCount int
)
for _, ch := range changes {
// Start new batch if this change pushes the current one over the limit.
size := changeSize(ch)
if len(batches) == 0 || batchSize+size > limit {
count := changeCount(ch)
size := changeSize(ch) * count
overSize := batchSize+size > sizeLimit
overCount := batchCount+count > countLimit
if len(batches) == 0 || overSize || overCount {
batches = append(batches, nil)
batchSize = 0
batchCount = 0
}
batches[len(batches)-1] = append(batches[len(batches)-1], ch)
batchSize += size
batchCount += count
}
return batches
}
@ -239,6 +251,13 @@ func changeSize(ch *route53.Change) int {
return size
}
func changeCount(ch *route53.Change) int {
if *ch.Action == "UPSERT" {
return 2
}
return 1
}
// collectRecords collects all TXT records below the given name.
func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) {
log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID))

16
cmd/devp2p/dns_route53_test.go
View file

@ -140,11 +140,23 @@ func TestRoute53ChangeSort(t *testing.T) {
t.Fatalf("wrong changes (got %d, want %d)", len(changes), len(wantChanges))
}
// Check splitting according to size.
wantSplit := [][]*route53.Change{
wantChanges[:4],
wantChanges[4:8],
wantChanges[4:6],
wantChanges[6:],
}
split := splitChanges(changes, 600)
split := splitChanges(changes, 600, 4000)
if !reflect.DeepEqual(split, wantSplit) {
t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit))
}
// Check splitting according to count.
wantSplit = [][]*route53.Change{
wantChanges[:5],
wantChanges[5:],
}
split = splitChanges(changes, 10000, 6)
if !reflect.DeepEqual(split, wantSplit) {
t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit))
}

4
cmd/devp2p/dnscmd.go
View file

@ -97,8 +97,8 @@ var (
)
const (
rootTTL = 1
treeNodeTTL = 2147483647
rootTTL = 30 * 60 // 30 min
treeNodeTTL = 4 * 7 * 24 * 60 * 60 // 4 weeks
)
// dnsSync performs dnsSyncCommand.

1
cmd/devp2p/main.go
View file

@ -59,6 +59,7 @@ func init() {
app.Commands = []cli.Command{
enrdumpCommand,
discv4Command,
discv5Command,
dnsCommand,
nodesetCommand,
}

2
cmd/devp2p/nodesetcmd.go
View file

@ -164,7 +164,7 @@ func ethFilter(args []string) (nodeFilter, error) {
case "goerli":
filter = forkid.NewStaticFilter(params.GoerliChainConfig, params.GoerliGenesisHash)
case "ropsten":
filter = forkid.NewStaticFilter(params.TestnetChainConfig, params.TestnetGenesisHash)
filter = forkid.NewStaticFilter(params.RopstenChainConfig, params.RopstenGenesisHash)
default:
return nil, fmt.Errorf("unknown network %q", args[0])
}

50
cmd/evm/runner.go
View file

@ -70,14 +70,13 @@ func readGenesis(genesisPath string) *core.Genesis {
return genesis
}
func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, uint64, time.Duration, error) {
var (
output []byte
gasLeft uint64
execTime time.Duration
err error
)
type execStats struct {
time time.Duration // The execution time.
allocs int64 // The number of heap allocations during execution.
bytesAllocated int64 // The cumulative number of bytes allocated during execution.
}
func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, gasLeft uint64, stats execStats, err error) {
if bench {
result := testing.Benchmark(func(b *testing.B) {
for i := 0; i < b.N; i++ {
@ -87,14 +86,21 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, uin
// Get the average execution time from the benchmarking result.
// There are other useful stats here that could be reported.
execTime = time.Duration(result.NsPerOp())
stats.time = time.Duration(result.NsPerOp())
stats.allocs = result.AllocsPerOp()
stats.bytesAllocated = result.AllocedBytesPerOp()
} else {
var memStatsBefore, memStatsAfter goruntime.MemStats
goruntime.ReadMemStats(&memStatsBefore)
startTime := time.Now()
output, gasLeft, err = execFunc()
execTime = time.Since(startTime)
stats.time = time.Since(startTime)
goruntime.ReadMemStats(&memStatsAfter)
stats.allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs)
stats.bytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc)
}
return output, gasLeft, execTime, err
return output, gasLeft, stats, err
}
func runCmd(ctx *cli.Context) error {
@ -129,10 +135,10 @@ func runCmd(ctx *cli.Context) error {
genesisConfig = gen
db := rawdb.NewMemoryDatabase()
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil)
chainConfig = gen.Config
} else {
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
genesisConfig = new(core.Genesis)
}
if ctx.GlobalString(SenderFlag.Name) != "" {
@ -256,7 +262,8 @@ func runCmd(ctx *cli.Context) error {
}
}
output, leftOverGas, execTime, err := timedExec(ctx.GlobalBool(BenchFlag.Name), execFunc)
bench := ctx.GlobalBool(BenchFlag.Name)
output, leftOverGas, stats, err := timedExec(bench, execFunc)
if ctx.GlobalBool(DumpFlag.Name) {
statedb.Commit(true)
@ -286,17 +293,12 @@ func runCmd(ctx *cli.Context) error {
vm.WriteLogs(os.Stderr, statedb.Logs())
}
if ctx.GlobalBool(StatDumpFlag.Name) {
var mem goruntime.MemStats
goruntime.ReadMemStats(&mem)
fmt.Fprintf(os.Stderr, `evm execution time: %v
heap objects: %d
allocations: %d
total allocations: %d
GC calls: %d
Gas used: %d
`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
if bench || ctx.GlobalBool(StatDumpFlag.Name) {
fmt.Fprintf(os.Stderr, `EVM gas used: %d
execution time: %v
allocations: %d
allocated bytes: %d
`, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated)
}
if tracer == nil {
fmt.Printf("0x%x\n", output)

2
cmd/evm/staterunner.go
View file

@ -96,7 +96,7 @@ func stateTestCmd(ctx *cli.Context) error {
for _, st := range test.Subtests() {
// Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
state, err := test.Run(st, cfg)
state, err := test.Run(st, cfg, false)
// print state root for evmlab tracing
if ctx.GlobalBool(MachineFlag.Name) && state != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))

31
cmd/geth/accountcmd_test.go
View file

@ -88,6 +88,37 @@ Path of the secret key file: .*UTC--.+--[0-9a-f]{40}
`)
}
func TestAccountImport(t *testing.T) {
tests := []struct{ key, output string }{
{
key: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
output: "Address: {fcad0b19bb29d4674531d6f115237e16afce377c}\n",
},
{
key: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef1",
output: "Fatal: Failed to load the private key: invalid character '1' at end of key file\n",
},
}
for _, test := range tests {
importAccountWithExpect(t, test.key, test.output)
}
}
func importAccountWithExpect(t *testing.T, key string, expected string) {
dir := tmpdir(t)
keyfile := filepath.Join(dir, "key.prv")
if err := ioutil.WriteFile(keyfile, []byte(key), 0600); err != nil {
t.Error(err)
}
passwordFile := filepath.Join(dir, "password.txt")
if err := ioutil.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil {
t.Error(err)
}
geth := runGeth(t, "account", "import", keyfile, "-password", passwordFile)
defer geth.ExpectExit()
geth.Expect(expected)
}
func TestAccountNewBadRepeat(t *testing.T) {
geth := runGeth(t, "account", "new", "--lightkdf")
defer geth.ExpectExit()

23
cmd/geth/chaincmd.go
View file

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/trie"
"gopkg.in/urfave/cli.v1"
)
@ -79,8 +80,17 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
utils.CacheFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.SnapshotFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.MetricsEnabledFlag,
utils.MetricsEnabledExpensiveFlag,
utils.MetricsEnableInfluxDBFlag,
utils.MetricsInfluxDBEndpointFlag,
utils.MetricsInfluxDBDatabaseFlag,
utils.MetricsInfluxDBUsernameFlag,
utils.MetricsInfluxDBPasswordFlag,
utils.MetricsInfluxDBTagsFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -146,8 +156,10 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
utils.CacheFlag,
utils.SyncModeFlag,
utils.FakePoWFlag,
utils.TestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.LegacyTestnetFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -193,9 +205,10 @@ Use "ethereum dump 0" to dump the genesis block.`,
utils.DataDirFlag,
utils.AncientFlag,
utils.CacheFlag,
utils.TestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.LegacyTestnetFlag,
utils.SyncModeFlag,
},
Category: "BLOCKCHAIN COMMANDS",
@ -254,6 +267,10 @@ func importChain(ctx *cli.Context) error {
if len(ctx.Args()) < 1 {
utils.Fatalf("This command requires an argument.")
}
// Start metrics export if enabled
utils.SetupMetrics(ctx)
// Start system runtime metrics collection
go metrics.CollectProcessMetrics(3 * time.Second)
stack := makeFullNode(ctx)
defer stack.Close()
@ -544,7 +561,7 @@ func dump(ctx *cli.Context) error {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb))
state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}

13
cmd/geth/consolecmd.go
View file

@ -123,10 +123,19 @@ func remoteConsole(ctx *cli.Context) error {
path = ctx.GlobalString(utils.DataDirFlag.Name)
}
if path != "" {
if ctx.GlobalBool(utils.TestnetFlag.Name) {
path = filepath.Join(path, "testnet")
if ctx.GlobalBool(utils.LegacyTestnetFlag.Name) || ctx.GlobalBool(utils.RopstenFlag.Name) {
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(path, "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
path = legacyPath
} else {
path = filepath.Join(path, "ropsten")
}
} else if ctx.GlobalBool(utils.RinkebyFlag.Name) {
path = filepath.Join(path, "rinkeby")
} else if ctx.GlobalBool(utils.GoerliFlag.Name) {
path = filepath.Join(path, "goerli")
}
}
endpoint = fmt.Sprintf("%s/geth.ipc", path)

28
cmd/geth/genesis_test.go
View file

@ -28,22 +28,6 @@ var customGenesisTests = []struct {
query string
result string
}{
// Plain genesis file without anything extra
{
genesis: `{
"alloc" : {},
"coinbase" : "0x0000000000000000000000000000000000000000",
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00"
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
},
// Genesis file with an empty chain configuration (ensure missing fields work)
{
genesis: `{
@ -52,14 +36,14 @@ var customGenesisTests = []struct {
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"nonce" : "0x0000000000001338",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00",
"config" : {}
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
result: "0x0000000000001338",
},
// Genesis file with specific chain configurations
{
@ -69,7 +53,7 @@ var customGenesisTests = []struct {
"difficulty" : "0x20000",
"extraData" : "",
"gasLimit" : "0x2fefd8",
"nonce" : "0x0000000000000042",
"nonce" : "0x0000000000001339",
"mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
"timestamp" : "0x00",
@ -80,7 +64,7 @@ var customGenesisTests = []struct {
}
}`,
query: "eth.getBlock(0).nonce",
result: "0x0000000000000042",
result: "0x0000000000001339",
},
}
@ -97,10 +81,10 @@ func TestCustomGenesis(t *testing.T) {
if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
t.Fatalf("test %d: failed to write genesis file: %v", i, err)
}
runGeth(t, "--datadir", datadir, "init", json).WaitExit()
runGeth(t, "--nousb", "--datadir", datadir, "init", json).WaitExit()
// Query the custom genesis block
geth := runGeth(t,
geth := runGeth(t, "--nousb",
"--datadir", datadir, "--maxpeers", "0", "--port", "0",
"--nodiscover", "--nat", "none", "--ipcdisable",
"--exec", tt.query, "console")

31
cmd/geth/main.go
View file

@ -74,9 +74,11 @@ var (
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,
utils.EthashDatasetsOnDiskFlag,
utils.EthashDatasetsLockMmapFlag,
utils.TxPoolLocalsFlag,
utils.TxPoolNoLocalsFlag,
utils.TxPoolJournalFlag,
@ -91,6 +93,7 @@ var (
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
utils.SnapshotFlag,
utils.LightServeFlag,
utils.LightLegacyServFlag,
utils.LightIngressFlag,
@ -106,6 +109,7 @@ var (
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.CacheSnapshotFlag,
utils.CacheNoPrefetchFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
@ -134,7 +138,8 @@ var (
utils.DNSDiscoveryFlag,
utils.DeveloperFlag,
utils.DeveloperPeriodFlag,
utils.TestnetFlag,
utils.LegacyTestnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.VMEnableDebugFlag,
@ -254,10 +259,32 @@ func main() {
// prepare manipulates memory cache allowance and setups metric system.
// This function should be called before launching devp2p stack.
func prepare(ctx *cli.Context) {
// If we're running a known preset, log it for convenience.
switch {
case ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name):
log.Info("Starting Geth on Ropsten testnet...")
log.Warn("The --testnet flag is ambiguous! Please specify one of --goerli, --rinkeby, or --ropsten.")
log.Warn("The generic --testnet flag is deprecated and will be removed in the future!")
case ctx.GlobalIsSet(utils.RopstenFlag.Name):
log.Info("Starting Geth on Ropsten testnet...")
case ctx.GlobalIsSet(utils.RinkebyFlag.Name):
log.Info("Starting Geth on Rinkeby testnet...")
case ctx.GlobalIsSet(utils.GoerliFlag.Name):
log.Info("Starting Geth on Görli testnet...")
case ctx.GlobalIsSet(utils.DeveloperFlag.Name):
log.Info("Starting Geth in ephemeral dev mode...")
case !ctx.GlobalIsSet(utils.NetworkIdFlag.Name):
log.Info("Starting Geth on Ethereum mainnet...")
}
// If we're a full node on mainnet without --cache specified, bump default cache allowance
if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
// Make sure we're not on any supported preconfigured testnet either
if !ctx.GlobalIsSet(utils.TestnetFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
if !ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name) && !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
// Nope, we're really on mainnet. Bump that cache up!
log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))

22
cmd/geth/retesteth.go
View file

@ -387,11 +387,13 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
inner = ethash.NewFaker()
case "Ethash":
inner = ethash.New(ethash.Config{
CacheDir: "ethash",
CachesInMem: 2,
CachesOnDisk: 3,
DatasetsInMem: 1,
DatasetsOnDisk: 2,
CacheDir: "ethash",
CachesInMem: 2,
CachesOnDisk: 3,
CachesLockMmap: false,
DatasetsInMem: 1,
DatasetsOnDisk: 2,
DatasetsLockMmap: false,
}, nil, false)
default:
return false, fmt.Errorf("unrecognised seal engine: %s", chainParams.SealEngine)
@ -888,6 +890,14 @@ func retesteth(ctx *cli.Context) error {
vhosts := splitAndTrim(ctx.GlobalString(utils.RPCVirtualHostsFlag.Name))
cors := splitAndTrim(ctx.GlobalString(utils.RPCCORSDomainFlag.Name))
// register apis and create handler stack
srv := rpc.NewServer()
err := node.RegisterApisFromWhitelist(rpcAPI, []string{"test", "eth", "debug", "web3"}, srv, false)
if err != nil {
utils.Fatalf("Could not register RPC apis: %w", err)
}
handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
// start http server
var RetestethHTTPTimeouts = rpc.HTTPTimeouts{
ReadTimeout: 120 * time.Second,
@ -895,7 +905,7 @@ func retesteth(ctx *cli.Context) error {
IdleTimeout: 120 * time.Second,
}
httpEndpoint := fmt.Sprintf("%s:%d", ctx.GlobalString(utils.RPCListenAddrFlag.Name), ctx.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"test", "eth", "debug", "web3"}, cors, vhosts, RetestethHTTPTimeouts)
listener, err := node.StartHTTPEndpoint(httpEndpoint, RetestethHTTPTimeouts, handler)
if err != nil {
utils.Fatalf("Could not start RPC api: %v", err)
}

8
cmd/geth/usage.go
View file

@ -72,9 +72,9 @@ var AppHelpFlagGroups = []flagGroup{
utils.NoUSBFlag,
utils.SmartCardDaemonPathFlag,
utils.NetworkIdFlag,
utils.TestnetFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.RinkebyFlag,
utils.RopstenFlag,
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
@ -109,9 +109,11 @@ var AppHelpFlagGroups = []flagGroup{
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,
utils.EthashDatasetsOnDiskFlag,
utils.EthashDatasetsLockMmapFlag,
},
},
{
@ -137,6 +139,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.CacheSnapshotFlag,
utils.CacheNoPrefetchFlag,
},
},
@ -242,6 +245,7 @@ var AppHelpFlagGroups = []flagGroup{
{
Name: "DEPRECATED",
Flags: []cli.Flag{
utils.LegacyTestnetFlag,
utils.LightLegacyServFlag,
utils.LightLegacyPeersFlag,
utils.MinerLegacyThreadsFlag,

4
cmd/puppeth/module_wallet.go
View file

@ -182,11 +182,11 @@ func checkWallet(client *sshClient, network string) (*walletInfos, error) {
// Run a sanity check to see if the devp2p and RPC ports are reachable
nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
if err = checkPort(client.server, nodePort); err != nil {
log.Warn(fmt.Sprintf("Wallet devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
log.Warn("Wallet devp2p port seems unreachable", "server", client.server, "port", nodePort, "err", err)
}
rpcPort := infos.portmap["8545/tcp"]
if err = checkPort(client.server, rpcPort); err != nil {
log.Warn(fmt.Sprintf("Wallet RPC port seems unreachable"), "server", client.server, "port", rpcPort, "err", err)
log.Warn("Wallet RPC port seems unreachable", "server", client.server, "port", rpcPort, "err", err)
}
// Assemble and return the useful infos
stats := &walletInfos{

2
cmd/utils/cmd.go
View file

@ -301,7 +301,7 @@ func ExportPreimages(db ethdb.Database, fn string) error {
defer writer.(*gzip.Writer).Close()
}
// Iterate over the preimages and export them
it := db.NewIteratorWithPrefix([]byte("secure-key-"))
it := db.NewIterator([]byte("secure-key-"), nil)
defer it.Release()
for it.Next() {

108
cmd/utils/flags.go
View file

@ -162,20 +162,24 @@ var (
}
NetworkIdFlag = cli.Uint64Flag{
Name: "networkid",
Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby)",
Usage: "Network identifier (integer, 1=Frontier, 3=Ropsten, 4=Rinkeby, 5=Görli)",
Value: eth.DefaultConfig.NetworkId,
}
TestnetFlag = cli.BoolFlag{
LegacyTestnetFlag = cli.BoolFlag{ // TODO(q9f): Remove after Ropsten is discontinued.
Name: "testnet",
Usage: "Ropsten network: pre-configured proof-of-work test network",
Usage: "Pre-configured test network (Deprecated: Please choose one of --goerli, --rinkeby, or --ropsten.)",
}
GoerliFlag = cli.BoolFlag{
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
}
RinkebyFlag = cli.BoolFlag{
Name: "rinkeby",
Usage: "Rinkeby network: pre-configured proof-of-authority test network",
}
GoerliFlag = cli.BoolFlag{
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
RopstenFlag = cli.BoolFlag{
Name: "ropsten",
Usage: "Ropsten network: pre-configured proof-of-work test network",
}
DeveloperFlag = cli.BoolFlag{
Name: "dev",
@ -225,6 +229,10 @@ var (
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
}
SnapshotFlag = cli.BoolFlag{
Name: "snapshot",
Usage: `Enables snapshot-database mode -- experimental work in progress feature`,
}
LightKDFFlag = cli.BoolFlag{
Name: "lightkdf",
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@ -301,6 +309,10 @@ var (
Usage: "Number of recent ethash caches to keep on disk (16MB each)",
Value: eth.DefaultConfig.Ethash.CachesOnDisk,
}
EthashCachesLockMmapFlag = cli.BoolFlag{
Name: "ethash.cacheslockmmap",
Usage: "Lock memory maps of recent ethash caches",
}
EthashDatasetDirFlag = DirectoryFlag{
Name: "ethash.dagdir",
Usage: "Directory to store the ethash mining DAGs",
@ -316,6 +328,10 @@ var (
Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)",
Value: eth.DefaultConfig.Ethash.DatasetsOnDisk,
}
EthashDatasetsLockMmapFlag = cli.BoolFlag{
Name: "ethash.dagslockmmap",
Usage: "Lock memory maps for recent ethash mining DAGs",
}
// Transaction pool settings
TxPoolLocalsFlag = cli.StringFlag{
Name: "txpool.locals",
@ -383,14 +399,19 @@ var (
}
CacheTrieFlag = cli.IntFlag{
Name: "cache.trie",
Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)",
Value: 25,
Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)",
Value: 15,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
Value: 25,
}
CacheSnapshotFlag = cli.IntFlag{
Name: "cache.snapshot",
Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)",
Value: 10,
}
CacheNoPrefetchFlag = cli.BoolFlag{
Name: "cache.noprefetch",
Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)",
@ -742,7 +763,6 @@ var (
Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements",
Value: "host=localhost",
}
EWASMInterpreterFlag = cli.StringFlag{
Name: "vm.ewasm",
Usage: "External ewasm configuration (default = built-in interpreter)",
@ -757,11 +777,17 @@ var (
// MakeDataDir retrieves the currently requested data directory, terminating
// if none (or the empty string) is specified. If the node is starting a testnet,
// the a subdirectory of the specified datadir will be used.
// then a subdirectory of the specified datadir will be used.
func MakeDataDir(ctx *cli.Context) string {
if path := ctx.GlobalString(DataDirFlag.Name); path != "" {
if ctx.GlobalBool(TestnetFlag.Name) {
return filepath.Join(path, "testnet")
if ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name) {
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(path, "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
return legacyPath
}
return filepath.Join(path, "ropsten")
}
if ctx.GlobalBool(RinkebyFlag.Name) {
return filepath.Join(path, "rinkeby")
@ -819,8 +845,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
} else {
urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name))
}
case ctx.GlobalBool(TestnetFlag.Name):
urls = params.TestnetBootnodes
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
urls = params.RopstenBootnodes
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
@ -1223,8 +1249,16 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
case ctx.GlobalBool(DeveloperFlag.Name):
cfg.DataDir = "" // unless explicitly requested, use memory databases
case ctx.GlobalBool(TestnetFlag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
case (ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name)) && cfg.DataDir == node.DefaultDataDir():
// Maintain compatibility with older Geth configurations storing the
// Ropsten database in `testnet` instead of `ropsten`.
legacyPath := filepath.Join(node.DefaultDataDir(), "testnet")
if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
log.Warn("Using the deprecated `testnet` datadir. Future versions will store the Ropsten chain in `ropsten`.")
cfg.DataDir = legacyPath
} else {
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten")
}
case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir():
@ -1297,12 +1331,18 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) {
if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) {
cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
}
if ctx.GlobalIsSet(EthashCachesLockMmapFlag.Name) {
cfg.Ethash.CachesLockMmap = ctx.GlobalBool(EthashCachesLockMmapFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) {
cfg.Ethash.DatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) {
cfg.Ethash.DatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetsLockMmapFlag.Name) {
cfg.Ethash.DatasetsLockMmap = ctx.GlobalBool(EthashDatasetsLockMmapFlag.Name)
}
}
func setMiner(ctx *cli.Context, cfg *miner.Config) {
@ -1418,7 +1458,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
CheckExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag)
CheckExclusive(ctx, LightLegacyServFlag, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
@ -1463,6 +1503,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
}
if !ctx.GlobalIsSet(SnapshotFlag.Name) {
cfg.SnapshotCache = 0 // Disabled
}
if ctx.GlobalIsSet(DocRootFlag.Name) {
cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name)
}
@ -1492,12 +1538,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Override any default configs for hard coded networks.
switch {
case ctx.GlobalBool(TestnetFlag.Name):
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 3
}
cfg.Genesis = core.DefaultTestnetGenesisBlock()
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.TestnetGenesisHash])
cfg.Genesis = core.DefaultRopstenGenesisBlock()
setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.RopstenGenesisHash])
case ctx.GlobalBool(RinkebyFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 4
@ -1679,8 +1725,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
func MakeGenesis(ctx *cli.Context) *core.Genesis {
var genesis *core.Genesis
switch {
case ctx.GlobalBool(TestnetFlag.Name):
genesis = core.DefaultTestnetGenesisBlock()
case ctx.GlobalBool(LegacyTestnetFlag.Name) || ctx.GlobalBool(RopstenFlag.Name):
genesis = core.DefaultRopstenGenesisBlock()
case ctx.GlobalBool(RinkebyFlag.Name):
genesis = core.DefaultRinkebyGenesisBlock()
case ctx.GlobalBool(GoerliFlag.Name):
@ -1706,12 +1752,14 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
engine = ethash.NewFaker()
if !ctx.GlobalBool(FakePoWFlag.Name) {
engine = ethash.New(ethash.Config{
CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
CachesInMem: eth.DefaultConfig.Ethash.CachesInMem,
CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk,
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,
DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
CachesInMem: eth.DefaultConfig.Ethash.CachesInMem,
CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk,
CachesLockMmap: eth.DefaultConfig.Ethash.CachesLockMmap,
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,
DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
DatasetsLockMmap: eth.DefaultConfig.Ethash.DatasetsLockMmap,
}, nil, false)
}
}
@ -1724,6 +1772,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
SnapshotLimit: eth.DefaultConfig.SnapshotCache,
}
if !ctx.GlobalIsSet(SnapshotFlag.Name) {
cache.SnapshotLimit = 0 // Disabled
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100

1
cmd/wnode/main.go
View file

@ -599,6 +599,7 @@ func messageLoop() {
}
ticker := time.NewTicker(time.Millisecond * 50)
defer ticker.Stop()
for {
select {

11
common/bytes.go
View file

@ -145,3 +145,14 @@ func TrimLeftZeroes(s []byte) []byte {
}
return s[idx:]
}
// TrimRightZeroes returns a subslice of s without trailing zeroes
func TrimRightZeroes(s []byte) []byte {
idx := len(s)
for ; idx > 0; idx-- {
if s[idx-1] != 0 {
break
}
}
return s[:idx]
}

19
common/bytes_test.go
View file

@ -105,3 +105,22 @@ func TestNoPrefixShortHexOddLength(t *testing.T) {
t.Errorf("Expected %x got %x", expected, result)
}
}
func TestTrimRightZeroes(t *testing.T) {
tests := []struct {
arr []byte
exp []byte
}{
{FromHex("0x00ffff00ff0000"), FromHex("0x00ffff00ff")},
{FromHex("0x00000000000000"), []byte{}},
{FromHex("0xff"), FromHex("0xff")},
{[]byte{}, []byte{}},
{FromHex("0x00ffffffffffff"), FromHex("0x00ffffffffffff")},
}
for i, test := range tests {
got := TrimRightZeroes(test.arr)
if !bytes.Equal(got, test.exp) {
t.Errorf("test %d, got %x exp %x", i, got, test.exp)
}
}
}

19
common/prque/lazyqueue_test.go
View file

@ -74,17 +74,22 @@ func TestLazyQueue(t *testing.T) {
q.Push(&items[i])
}
var lock sync.Mutex
stopCh := make(chan chan struct{})
var (
lock sync.Mutex
wg sync.WaitGroup
stopCh = make(chan chan struct{})
)
defer wg.Wait()
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-clock.After(testQueueRefresh):
lock.Lock()
q.Refresh()
lock.Unlock()
case stop := <-stopCh:
close(stop)
case <-stopCh:
return
}
}
@ -104,6 +109,8 @@ func TestLazyQueue(t *testing.T) {
if rand.Intn(100) == 0 {
p := q.PopItem().(*lazyItem)
if p.p != maxPri {
lock.Unlock()
close(stopCh)
t.Fatalf("incorrect item (best known priority %d, popped %d)", maxPri, p.p)
}
q.Push(p)
@ -113,7 +120,5 @@ func TestLazyQueue(t *testing.T) {
clock.WaitForTimers(1)
}
stop := make(chan struct{})
stopCh <- stop
<-stop
close(stopCh)
}

27
consensus/ethash/algorithm_test.go
View file

@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
go func(idx int) {
defer pend.Done()
ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal, nil}, nil, false)
ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
defer ethash.Close()
if err := ethash.VerifySeal(nil, block.Header()); err != nil {
t.Errorf("proc %d: block verification failed: %v", idx, err)
@ -787,3 +787,28 @@ func BenchmarkHashimotoFullSmall(b *testing.B) {
hashimotoFull(dataset, hash, 0)
}
}
func benchmarkHashimotoFullMmap(b *testing.B, name string, lock bool) {
b.Run(name, func(b *testing.B) {
tmpdir, err := ioutil.TempDir("", "ethash-test")
if err != nil {
b.Fatal(err)
}
defer os.RemoveAll(tmpdir)
d := &dataset{epoch: 0}
d.generate(tmpdir, 1, lock, false)
var hash [common.HashLength]byte
b.ResetTimer()
for i := 0; i < b.N; i++ {
binary.PutVarint(hash[:], int64(i))
hashimotoFull(d.dataset, hash[:], 0)
}
})
}
// Benchmarks the full verification performance for mmap
func BenchmarkHashimotoFullMmap(b *testing.B) {
benchmarkHashimotoFullMmap(b, "WithLock", true)
benchmarkHashimotoFullMmap(b, "WithoutLock", false)
}

59
consensus/ethash/ethash.go
View file

@ -48,7 +48,7 @@ var (
two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
// sharedEthash is a full instance that can be shared between multiple users.
sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal, nil}, nil, false)
sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false)
// algorithmRevision is the data structure version used for file naming.
algorithmRevision = 23
@ -65,7 +65,7 @@ func isLittleEndian() bool {
}
// memoryMap tries to memory map a file of uint32s for read only access.
func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) {
func memoryMap(path string, lock bool) (*os.File, mmap.MMap, []uint32, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0644)
if err != nil {
return nil, nil, nil, err
@ -82,6 +82,13 @@ func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) {
return nil, nil, nil, ErrInvalidDumpMagic
}
}
if lock {
if err := mem.Lock(); err != nil {
mem.Unmap()
file.Close()
return nil, nil, nil, err
}
}
return file, mem, buffer[len(dumpMagic):], err
}
@ -107,7 +114,7 @@ func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) {
// memoryMapAndGenerate tries to memory map a temporary file of uint32s for write
// access, fill it with the data from a generator and then move it into the final
// path requested.
func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) {
func memoryMapAndGenerate(path string, size uint64, lock bool, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) {
// Ensure the data folder exists
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return nil, nil, nil, err
@ -142,7 +149,7 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint
if err := os.Rename(temp, path); err != nil {
return nil, nil, nil, err
}
return memoryMap(path)
return memoryMap(path, lock)
}
// lru tracks caches or datasets by their last use time, keeping at most N of them.
@ -213,7 +220,7 @@ func newCache(epoch uint64) interface{} {
}
// generate ensures that the cache content is generated before use.
func (c *cache) generate(dir string, limit int, test bool) {
func (c *cache) generate(dir string, limit int, lock bool, test bool) {
c.once.Do(func() {
size := cacheSize(c.epoch*epochLength + 1)
seed := seedHash(c.epoch*epochLength + 1)
@ -240,7 +247,7 @@ func (c *cache) generate(dir string, limit int, test bool) {
// Try to load the file from disk and memory map it
var err error
c.dump, c.mmap, c.cache, err = memoryMap(path)
c.dump, c.mmap, c.cache, err = memoryMap(path, lock)
if err == nil {
logger.Debug("Loaded old ethash cache from disk")
return
@ -248,7 +255,7 @@ func (c *cache) generate(dir string, limit int, test bool) {
logger.Debug("Failed to load old ethash cache", "err", err)
// No previous cache available, create a new cache file to fill
c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) })
c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, lock, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) })
if err != nil {
logger.Error("Failed to generate mapped ethash cache", "err", err)
@ -290,7 +297,7 @@ func newDataset(epoch uint64) interface{} {
}
// generate ensures that the dataset content is generated before use.
func (d *dataset) generate(dir string, limit int, test bool) {
func (d *dataset) generate(dir string, limit int, lock bool, test bool) {
d.once.Do(func() {
// Mark the dataset generated after we're done. This is needed for remote
defer atomic.StoreUint32(&d.done, 1)
@ -326,7 +333,7 @@ func (d *dataset) generate(dir string, limit int, test bool) {
// Try to load the file from disk and memory map it
var err error
d.dump, d.mmap, d.dataset, err = memoryMap(path)
d.dump, d.mmap, d.dataset, err = memoryMap(path, lock)
if err == nil {
logger.Debug("Loaded old ethash dataset from disk")
return
@ -337,7 +344,7 @@ func (d *dataset) generate(dir string, limit int, test bool) {
cache := make([]uint32, csize/4)
generateCache(cache, d.epoch, seed)
d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) })
d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, lock, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) })
if err != nil {
logger.Error("Failed to generate mapped ethash dataset", "err", err)
@ -372,13 +379,13 @@ func (d *dataset) finalizer() {
// MakeCache generates a new ethash cache and optionally stores it to disk.
func MakeCache(block uint64, dir string) {
c := cache{epoch: block / epochLength}
c.generate(dir, math.MaxInt32, false)
c.generate(dir, math.MaxInt32, false, false)
}
// MakeDataset generates a new ethash dataset and optionally stores it to disk.
func MakeDataset(block uint64, dir string) {
d := dataset{epoch: block / epochLength}
d.generate(dir, math.MaxInt32, false)
d.generate(dir, math.MaxInt32, false, false)
}
// Mode defines the type and amount of PoW verification an ethash engine makes.
@ -394,13 +401,15 @@ const (
// Config are the configuration parameters of the ethash.
type Config struct {
CacheDir string
CachesInMem int
CachesOnDisk int
DatasetDir string
DatasetsInMem int
DatasetsOnDisk int
PowMode Mode
CacheDir string
CachesInMem int
CachesOnDisk int
CachesLockMmap bool
DatasetDir string
DatasetsInMem int
DatasetsOnDisk int
DatasetsLockMmap bool
PowMode Mode
Log log.Logger `toml:"-"`
}
@ -549,12 +558,12 @@ func (ethash *Ethash) cache(block uint64) *cache {
current := currentI.(*cache)
// Wait for generation finish.
current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
// If we need a new future cache, now's a good time to regenerate it.
if futureI != nil {
future := futureI.(*cache)
go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest)
go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
}
return current
}
@ -574,20 +583,20 @@ func (ethash *Ethash) dataset(block uint64, async bool) *dataset {
// If async is specified, generate everything in a background thread
if async && !current.generated() {
go func() {
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
if futureI != nil {
future := futureI.(*dataset)
future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
}
}()
} else {
// Either blocking generation was requested, or already done
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
if futureI != nil {
future := futureI.(*dataset)
go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest)
go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.DatasetsLockMmap, ethash.config.PowMode == ModeTest)
}
}
return current

85
console/console.go
View file

@ -340,62 +340,61 @@ func (c *Console) Evaluate(statement string) {
// the configured user prompter.
func (c *Console) Interactive() {
var (
prompt = c.prompt // Current prompt line (used for multi-line inputs)
indents = 0 // Current number of input indents (used for multi-line inputs)
input = "" // Current user input
scheduler = make(chan string) // Channel to send the next prompt on and receive the input
prompt = c.prompt // the current prompt line (used for multi-line inputs)
indents = 0 // the current number of input indents (used for multi-line inputs)
input = "" // the current user input
inputLine = make(chan string, 1) // receives user input
inputErr = make(chan error, 1) // receives liner errors
requestLine = make(chan string) // requests a line of input
interrupt = make(chan os.Signal, 1)
)
// Start a goroutine to listen for prompt requests and send back inputs
go func() {
for {
// Read the next user input
line, err := c.prompter.PromptInput(<-scheduler)
if err != nil {
// In case of an error, either clear the prompt or fail
if err == liner.ErrPromptAborted { // ctrl-C
prompt, indents, input = c.prompt, 0, ""
scheduler <- ""
continue
}
close(scheduler)
return
}
// User input retrieved, send for interpretation and loop
scheduler <- line
}
}()
// Monitor Ctrl-C too in case the input is empty and we need to bail
abort := make(chan os.Signal, 1)
signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
// Start sending prompts to the user and reading back inputs
// Monitor Ctrl-C. While liner does turn on the relevant terminal mode bits to avoid
// the signal, a signal can still be received for unsupported terminals. Unfortunately
// there is no way to cancel the line reader when this happens. The readLines
// goroutine will be leaked in this case.
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(interrupt)
// The line reader runs in a separate goroutine.
go c.readLines(inputLine, inputErr, requestLine)
defer close(requestLine)
for {
// Send the next prompt, triggering an input read and process the result
scheduler <- prompt
// Send the next prompt, triggering an input read.
requestLine <- prompt
select {
case <-abort:
// User forcefully quite the console
case <-interrupt:
fmt.Fprintln(c.printer, "caught interrupt, exiting")
return
case line, ok := <-scheduler:
// User input was returned by the prompter, handle special cases
if !ok || (indents <= 0 && exit.MatchString(line)) {
case err := <-inputErr:
if err == liner.ErrPromptAborted && indents > 0 {
// When prompting for multi-line input, the first Ctrl-C resets
// the multi-line state.
prompt, indents, input = c.prompt, 0, ""
continue
}
return
case line := <-inputLine:
// User input was returned by the prompter, handle special cases.
if indents <= 0 && exit.MatchString(line) {
return
}
if onlyWhitespace.MatchString(line) {
continue
}
// Append the line to the input and check for multi-line interpretation
// Append the line to the input and check for multi-line interpretation.
input += line + "\n"
indents = countIndents(input)
if indents <= 0 {
prompt = c.prompt
} else {
prompt = strings.Repeat(".", indents*3) + " "
}
// If all the needed lines are present, save the command and run
// If all the needed lines are present, save the command and run it.
if indents <= 0 {
if len(input) > 0 && input[0] != ' ' && !passwordRegexp.MatchString(input) {
if command := strings.TrimSpace(input); len(c.history) == 0 || command != c.history[len(c.history)-1] {
@ -412,6 +411,18 @@ func (c *Console) Interactive() {
}
}
// readLines runs in its own goroutine, prompting for input.
func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-chan string) {
for p := range prompt {
line, err := c.prompter.PromptInput(p)
if err != nil {
errc <- err
} else {
input <- line
}
}
}
// countIndents returns the number of identations for the given input.
// In case of invalid input such as var a = } the result can be negative.
func countIndents(input string) int {

82
core/blockchain.go
View file

@ -34,6 +34,7 @@ import (
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb"
@ -61,6 +62,10 @@ var (
storageUpdateTimer = metrics.NewRegisteredTimer("chain/storage/updates", nil)
storageCommitTimer = metrics.NewRegisteredTimer("chain/storage/commits", nil)
snapshotAccountReadTimer = metrics.NewRegisteredTimer("chain/snapshot/account/reads", nil)
snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil)
snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil)
blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil)
@ -115,6 +120,9 @@ type CacheConfig struct {
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node)
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
}
// BlockChain represents the canonical chain given a database with a genesis
@ -136,6 +144,7 @@ type BlockChain struct {
cacheConfig *CacheConfig // Cache configuration for pruning
db ethdb.Database // Low level persistent database to store final content in
snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping
@ -188,6 +197,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 256,
SnapshotWait: true,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
@ -293,6 +304,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
}
}
}
// Load any existing snapshot, regenerating it if loading failed
if bc.cacheConfig.SnapshotLimit > 0 {
bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, bc.CurrentBlock().Root(), !bc.cacheConfig.SnapshotWait)
}
// Take ownership of this particular state
go bc.update()
return bc, nil
@ -339,7 +354,7 @@ func (bc *BlockChain) loadLastState() error {
return bc.Reset()
}
// Make sure the state associated with the block is available
if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
if _, err := state.New(currentBlock.Root(), bc.stateCache, bc.snaps); err != nil {
// Dangling block without a state associated, init from scratch
log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
if err := bc.repair(&currentBlock); err != nil {
@ -401,7 +416,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
if newHeadBlock == nil {
newHeadBlock = bc.genesisBlock
} else {
if _, err := state.New(newHeadBlock.Root(), bc.stateCache); err != nil {
if _, err := state.New(newHeadBlock.Root(), bc.stateCache, bc.snaps); err != nil {
// Rewound state missing, rolled back to before pivot, reset to genesis
newHeadBlock = bc.genesisBlock
}
@ -486,6 +501,10 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
headBlockGauge.Update(int64(block.NumberU64()))
bc.chainmu.Unlock()
// Destroy any existing state snapshot and regenerate it in the background
if bc.snaps != nil {
bc.snaps.Rebuild(block.Root())
}
log.Info("Committed new head block", "number", block.Number(), "hash", hash)
return nil
}
@ -501,6 +520,15 @@ func (bc *BlockChain) CurrentBlock() *types.Block {
return bc.currentBlock.Load().(*types.Block)
}
// Snapshot returns the blockchain snapshot tree. This method is mainly used for
// testing, to make it possible to verify the snapshot after execution.
//
// Warning: There are no guarantees about the safety of using the returned 'snap' if the
// blockchain is simultaneously importing blocks, so take care.
func (bc *BlockChain) Snapshot() *snapshot.Tree {
return bc.snaps
}
// CurrentFastBlock retrieves the current fast-sync head block of the canonical
// chain. The block is retrieved from the blockchain's internal cache.
func (bc *BlockChain) CurrentFastBlock() *types.Block {
@ -524,7 +552,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) {
// StateAt returns a new mutable state based on a particular point in time.
func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.stateCache)
return state.New(root, bc.stateCache, bc.snaps)
}
// StateCache returns the caching database underpinning the blockchain instance.
@ -576,7 +604,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
func (bc *BlockChain) repair(head **types.Block) error {
for {
// Abort if we've rewound to a head block that does have associated state
if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
if _, err := state.New((*head).Root(), bc.stateCache, bc.snaps); err == nil {
log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash())
return nil
}
@ -839,6 +867,14 @@ func (bc *BlockChain) Stop() {
bc.wg.Wait()
// Ensure that the entirety of the state snapshot is journalled to disk.
var snapBase common.Hash
if bc.snaps != nil {
var err error
if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil {
log.Error("Failed to journal state snapshot", "err", err)
}
}
// Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios:
// - HEAD: So we don't need to reprocess any blocks in the general case
@ -857,6 +893,12 @@ func (bc *BlockChain) Stop() {
}
}
}
if snapBase != (common.Hash{}) {
log.Info("Writing snapshot state to disk", "root", snapBase)
if err := triedb.Commit(snapBase, true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
}
}
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem().(common.Hash))
}
@ -864,7 +906,7 @@ func (bc *BlockChain) Stop() {
log.Error("Dangling trie nodes after full cleanup")
}
}
log.Info("Blockchain manager stopped")
log.Info("Blockchain stopped")
}
func (bc *BlockChain) procFutureBlocks() {
@ -1647,7 +1689,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
if parent == nil {
parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
}
statedb, err := state.New(parent.Root, bc.stateCache)
statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
if err != nil {
return it.index, err
}
@ -1656,9 +1698,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
var followupInterrupt uint32
if !bc.cacheConfig.TrieCleanNoPrefetch {
if followup, err := it.peek(); followup != nil && err == nil {
throwaway, _ := state.New(parent.Root, bc.stateCache)
throwaway, _ := state.New(parent.Root, bc.stateCache, bc.snaps)
go func(start time.Time, followup *types.Block, throwaway *state.StateDB, interrupt *uint32) {
bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, interrupt)
bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, &followupInterrupt)
blockPrefetchExecuteTimer.Update(time.Since(start))
if atomic.LoadUint32(interrupt) == 1 {
@ -1676,14 +1718,16 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
return it.index, err
}
// Update the metrics touched during block processing
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them
storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them
accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them
storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them
storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete, we can mark them
accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete, we can mark them
storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete, we can mark them
snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete, we can mark them
snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete, we can mark them
triehash := statedb.AccountHashes + statedb.StorageHashes // Save to not double count in validation
trieproc := statedb.AccountReads + statedb.AccountUpdates
trieproc += statedb.StorageReads + statedb.StorageUpdates
trieproc := statedb.SnapshotAccountReads + statedb.AccountReads + statedb.AccountUpdates
trieproc += statedb.SnapshotStorageReads + statedb.StorageReads + statedb.StorageUpdates
blockExecutionTimer.Update(time.Since(substart) - trieproc - triehash)
@ -1705,17 +1749,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
// Write the block to the chain and get the status.
substart = time.Now()
status, err := bc.writeBlockWithState(block, receipts, logs, statedb, false)
atomic.StoreUint32(&followupInterrupt, 1)
if err != nil {
atomic.StoreUint32(&followupInterrupt, 1)
return it.index, err
}
atomic.StoreUint32(&followupInterrupt, 1)
// Update the metrics touched during block commit
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits)
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits)
blockInsertTimer.UpdateSince(start)
switch status {

524
core/blockchain_test.go
View file

@ -144,7 +144,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
}
return err
}
statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache)
statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache, nil)
if err != nil {
return err
}
@ -973,6 +973,7 @@ func TestLogReorgs(t *testing.T) {
t.Fatalf("failed to insert forked chain: %v", err)
}
timeout := time.NewTimer(1 * time.Second)
defer timeout.Stop()
select {
case <-done:
case <-timeout.C:
@ -2315,7 +2316,7 @@ func TestDeleteCreateRevert(t *testing.T) {
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: []byte{byte(vm.PC), 0xFF},
Code: []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)},
Nonce: 1,
Balance: big.NewInt(0),
},
@ -2362,3 +2363,522 @@ func TestDeleteCreateRevert(t *testing.T) {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
}
// TestDeleteRecreateSlots tests a state-transition that contains both deletion
// and recreation of contract state.
// Contract A exists, has slots 1 and 2 set
// Tx 1: Selfdestruct A
// Tx 2: Re-create A, set slots 3 and 4
// Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct,
// and then the new slots exist
func TestDeleteRecreateSlots(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. Set slots 3=3, 4=4,
// 2. Return aaCode
initCode := []byte{
byte(vm.PUSH1), 0x3, // value
byte(vm.PUSH1), 0x3, // location
byte(vm.SSTORE), // Set slot[3] = 1
byte(vm.PUSH1), 0x4, // value
byte(vm.PUSH1), 0x4, // location
byte(vm.SSTORE), // Set slot[4] = 1
// Slots are set, now return the code
byte(vm.PUSH2), byte(vm.PC), byte(vm.SELFDESTRUCT), // Push code on stack
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
// Code is now in memory.
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), byte(32 - 2), // offset
byte(vm.RETURN),
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
// The contract BB recreates AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to AA, to kill it
tx, _ := types.SignTx(types.NewTransaction(0, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
// One transaction to BB, to recreate AA
tx, _ = types.SignTx(types.NewTransaction(1, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
Debug: true,
Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then slot 1 and 2 are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
// Also, 3 and 4 should be set
if got, exp := statedb.GetState(aa, common.HexToHash("03")), common.HexToHash("03"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("04")), common.HexToHash("04"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
}
}
// TestDeleteRecreateAccount tests a state-transition that contains deletion of a
// contract with storage, and a recreate of the same contract via a
// regular value-transfer
// Expected outcome is that _all_ slots are cleared from A
func TestDeleteRecreateAccount(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
},
}
genesis := gspec.MustCommit(db)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to AA, to kill it
tx, _ := types.SignTx(types.NewTransaction(0, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
// One transaction to AA, to recreate it (but without storage
tx, _ = types.SignTx(types.NewTransaction(1, aa,
big.NewInt(1), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
Debug: true,
Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then both slots are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
}
}
// TestDeleteRecreateSlotsAcrossManyBlocks tests multiple state-transition that contains both deletion
// and recreation of contract state.
// Contract A exists, has slots 1 and 2 set
// Tx 1: Selfdestruct A
// Tx 2: Re-create A, set slots 3 and 4
// Expected outcome is that _all_ slots are cleared from A, due to the selfdestruct,
// and then the new slots exist
func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
aaStorage = make(map[common.Hash]common.Hash) // Initial storage in AA
aaCode = []byte{byte(vm.PC), byte(vm.SELFDESTRUCT)} // Code for AA (simple selfdestruct)
)
// Populate two slots
aaStorage[common.HexToHash("01")] = common.HexToHash("01")
aaStorage[common.HexToHash("02")] = common.HexToHash("02")
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. Set slots 3=blocknum+1, 4=4,
// 2. Return aaCode
initCode := []byte{
byte(vm.PUSH1), 0x1, //
byte(vm.NUMBER), // value = number + 1
byte(vm.ADD), //
byte(vm.PUSH1), 0x3, // location
byte(vm.SSTORE), // Set slot[3] = number + 1
byte(vm.PUSH1), 0x4, // value
byte(vm.PUSH1), 0x4, // location
byte(vm.SSTORE), // Set slot[4] = 4
// Slots are set, now return the code
byte(vm.PUSH2), byte(vm.PC), byte(vm.SELFDESTRUCT), // Push code on stack
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
// Code is now in memory.
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), byte(32 - 2), // offset
byte(vm.RETURN),
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address 0xAAAAA selfdestructs if called
aa: {
// Code needs to just selfdestruct
Code: aaCode,
Nonce: 1,
Balance: big.NewInt(0),
Storage: aaStorage,
},
// The contract BB recreates AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
var nonce uint64
type expectation struct {
exist bool
blocknum int
values map[int]int
}
var current = &expectation{
exist: true, // exists in genesis
blocknum: 0,
values: map[int]int{1: 1, 2: 2},
}
var expectations []*expectation
var newDestruct = func(e *expectation) *types.Transaction {
tx, _ := types.SignTx(types.NewTransaction(nonce, aa,
big.NewInt(0), 50000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
nonce++
if e.exist {
e.exist = false
e.values = nil
}
t.Logf("block %d; adding destruct\n", e.blocknum)
return tx
}
var newResurrect = func(e *expectation) *types.Transaction {
tx, _ := types.SignTx(types.NewTransaction(nonce, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
nonce++
if !e.exist {
e.exist = true
e.values = map[int]int{3: e.blocknum + 1, 4: 4}
}
t.Logf("block %d; adding resurrect\n", e.blocknum)
return tx
}
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 150, func(i int, b *BlockGen) {
var exp = new(expectation)
exp.blocknum = i + 1
exp.values = make(map[int]int)
for k, v := range current.values {
exp.values[k] = v
}
exp.exist = current.exist
b.SetCoinbase(common.Address{1})
if i%2 == 0 {
b.AddTx(newDestruct(exp))
}
if i%3 == 0 {
b.AddTx(newResurrect(exp))
}
if i%5 == 0 {
b.AddTx(newDestruct(exp))
}
if i%7 == 0 {
b.AddTx(newResurrect(exp))
}
expectations = append(expectations, exp)
current = exp
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
//Debug: true,
//Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
var asHash = func(num int) common.Hash {
return common.BytesToHash([]byte{byte(num)})
}
for i, block := range blocks {
blockNum := i + 1
if n, err := chain.InsertChain([]*types.Block{block}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
statedb, _ := chain.State()
// If all is correct, then slot 1 and 2 are zero
if got, exp := statedb.GetState(aa, common.HexToHash("01")), (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
}
if got, exp := statedb.GetState(aa, common.HexToHash("02")), (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
}
exp := expectations[i]
if exp.exist {
if !statedb.Exist(aa) {
t.Fatalf("block %d, expected %v to exist, it did not", blockNum, aa)
}
for slot, val := range exp.values {
if gotValue, expValue := statedb.GetState(aa, asHash(slot)), asHash(val); gotValue != expValue {
t.Fatalf("block %d, slot %d, got %x exp %x", blockNum, slot, gotValue, expValue)
}
}
} else {
if statedb.Exist(aa) {
t.Fatalf("block %d, expected %v to not exist, it did", blockNum, aa)
}
}
}
}
// TestInitThenFailCreateContract tests a pretty notorious case that happened
// on mainnet over blocks 7338108, 7338110 and 7338115.
// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated
// with 0.001 ether (thus created but no code)
// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on
// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the
// deployment fails due to OOG during initcode execution
// - Block 7338115: another tx checks the balance of
// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as
// zero.
//
// The problem being that the snapshotter maintains a destructset, and adds items
// to the destructset in case something is created "onto" an existing item.
// We need to either roll back the snapDestructs, or not place it into snapDestructs
// in the first place.
//
func TestInitThenFailCreateContract(t *testing.T) {
var (
// Generate a canonical chain to act as the main dataset
engine = ethash.NewFaker()
db = rawdb.NewMemoryDatabase()
// A sender who makes transactions, has some funds
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb")
)
// The bb-code needs to CREATE2 the aa contract. It consists of
// both initcode and deployment code
// initcode:
// 1. If blocknum < 1, error out (e.g invalid opcode)
// 2. else, return a snippet of code
initCode := []byte{
byte(vm.PUSH1), 0x1, // y (2)
byte(vm.NUMBER), // x (number)
byte(vm.GT), // x > y?
byte(vm.PUSH1), byte(0x8),
byte(vm.JUMPI), // jump to label if number > 2
byte(0xFE), // illegal opcode
byte(vm.JUMPDEST),
byte(vm.PUSH1), 0x2, // size
byte(vm.PUSH1), 0x0, // offset
byte(vm.RETURN), // return 2 bytes of zero-code
}
if l := len(initCode); l > 32 {
t.Fatalf("init code is too long for a pushx, need a more elaborate deployer")
}
bbCode := []byte{
// Push initcode onto stack
byte(vm.PUSH1) + byte(len(initCode)-1)}
bbCode = append(bbCode, initCode...)
bbCode = append(bbCode, []byte{
byte(vm.PUSH1), 0x0, // memory start on stack
byte(vm.MSTORE),
byte(vm.PUSH1), 0x00, // salt
byte(vm.PUSH1), byte(len(initCode)), // size
byte(vm.PUSH1), byte(32 - len(initCode)), // offset
byte(vm.PUSH1), 0x00, // endowment
byte(vm.CREATE2),
}...)
initHash := crypto.Keccak256Hash(initCode)
aa := crypto.CreateAddress2(bb, [32]byte{}, initHash[:])
t.Logf("Destination address: %x\n", aa)
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{
address: {Balance: funds},
// The address aa has some funds
aa: {Balance: big.NewInt(100000)},
// The contract BB tries to create code onto AA
bb: {
Code: bbCode,
Balance: big.NewInt(1),
},
},
}
genesis := gspec.MustCommit(db)
nonce := uint64(0)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 4, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
// One transaction to BB
tx, _ := types.SignTx(types.NewTransaction(nonce, bb,
big.NewInt(0), 100000, big.NewInt(1), nil), types.HomesteadSigner{}, key)
b.AddTx(tx)
nonce++
})
// Import the canonical chain
diskdb := rawdb.NewMemoryDatabase()
gspec.MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{
//Debug: true,
//Tracer: vm.NewJSONLogger(nil, os.Stdout),
}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
statedb, _ := chain.State()
if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 {
t.Fatalf("Genesis err, got %v exp %v", got, exp)
}
// First block tries to create, but fails
{
block := blocks[0]
if _, err := chain.InsertChain([]*types.Block{blocks[0]}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
}
statedb, _ = chain.State()
if got, exp := statedb.GetBalance(aa), big.NewInt(100000); got.Cmp(exp) != 0 {
t.Fatalf("block %d: got %v exp %v", block.NumberU64(), got, exp)
}
}
// Import the rest of the blocks
for _, block := range blocks[1:] {
if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
}
}
}

2
core/chain_makers.go
View file

@ -228,7 +228,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
return nil, nil
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(db))
statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil)
if err != nil {
panic(err)
}

4
core/forkid/forkid_test.go
View file

@ -65,8 +65,8 @@ func TestCreation(t *testing.T) {
},
// Ropsten test cases
{
params.TestnetChainConfig,
params.TestnetGenesisHash,
params.RopstenChainConfig,
params.RopstenGenesisHash,
[]testcase{
{0, ID{Hash: checksumToBytes(0x30c7ddbc), Next: 10}}, // Unsynced, last Frontier, Homestead and first Tangerine block
{9, ID{Hash: checksumToBytes(0x30c7ddbc), Next: 10}}, // Last Tangerine block

20
core/genesis.go
View file

@ -178,7 +178,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
// We have the genesis block in database(perhaps in ancient database)
// but the corresponding state is missing.
header := rawdb.ReadHeader(db, stored, 0)
if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0)); err != nil {
if _, err := state.New(header.Root, state.NewDatabaseWithCache(db, 0), nil); err != nil {
if genesis == nil {
genesis = DefaultGenesisBlock()
}
@ -246,8 +246,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
return g.Config
case ghash == params.MainnetGenesisHash:
return params.MainnetChainConfig
case ghash == params.TestnetGenesisHash:
return params.TestnetChainConfig
case ghash == params.RopstenGenesisHash:
return params.RopstenChainConfig
case ghash == params.RinkebyGenesisHash:
return params.RinkebyChainConfig
case ghash == params.GoerliGenesisHash:
return params.GoerliChainConfig
default:
return params.AllEthashProtocolChanges
}
@ -259,7 +263,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
if db == nil {
db = rawdb.NewMemoryDatabase()
}
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db), nil)
for addr, account := range g.Alloc {
statedb.AddBalance(addr, account.Balance)
statedb.SetCode(addr, account.Code)
@ -347,15 +351,15 @@ func DefaultGenesisBlock() *Genesis {
}
}
// DefaultTestnetGenesisBlock returns the Ropsten network genesis block.
func DefaultTestnetGenesisBlock() *Genesis {
// DefaultRopstenGenesisBlock returns the Ropsten network genesis block.
func DefaultRopstenGenesisBlock() *Genesis {
return &Genesis{
Config: params.TestnetChainConfig,
Config: params.RopstenChainConfig,
Nonce: 66,
ExtraData: hexutil.MustDecode("0x3535353535353535353535353535353535353535353535353535353535353535"),
GasLimit: 16777216,
Difficulty: big.NewInt(1048576),
Alloc: decodePrealloc(testnetAllocData),
Alloc: decodePrealloc(ropstenAllocData),
}
}

2
core/genesis_alloc.go
View file

File diff suppressed because one or more lines are too long

16
core/genesis_test.go
View file

@ -35,9 +35,9 @@ func TestDefaultGenesisBlock(t *testing.T) {
if block.Hash() != params.MainnetGenesisHash {
t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash)
}
block = DefaultTestnetGenesisBlock().ToBlock(nil)
if block.Hash() != params.TestnetGenesisHash {
t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash)
block = DefaultRopstenGenesisBlock().ToBlock(nil)
if block.Hash() != params.RopstenGenesisHash {
t.Errorf("wrong ropsten genesis hash, got %v, want %v", block.Hash(), params.RopstenGenesisHash)
}
}
@ -95,14 +95,14 @@ func TestSetupGenesis(t *testing.T) {
wantConfig: customg.Config,
},
{
name: "custom block in DB, genesis == testnet",
name: "custom block in DB, genesis == ropsten",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
customg.MustCommit(db)
return SetupGenesisBlock(db, DefaultTestnetGenesisBlock())
return SetupGenesisBlock(db, DefaultRopstenGenesisBlock())
},
wantErr: &GenesisMismatchError{Stored: customghash, New: params.TestnetGenesisHash},
wantHash: params.TestnetGenesisHash,
wantConfig: params.TestnetChainConfig,
wantErr: &GenesisMismatchError{Stored: customghash, New: params.RopstenGenesisHash},
wantHash: params.RopstenGenesisHash,
wantConfig: params.RopstenChainConfig,
},
{
name: "compatible config in DB",

2
core/rawdb/accessors_chain.go
View file

@ -69,7 +69,7 @@ func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash {
prefix := headerKeyPrefix(number)
hashes := make([]common.Hash, 0, 1)
it := db.NewIteratorWithPrefix(prefix)
it := db.NewIterator(prefix, nil)
defer it.Release()
for it.Next() {

120
core/rawdb/accessors_snapshot.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
// ReadSnapshotRoot retrieves the root of the block whose state is contained in
// the persisted snapshot.
func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash {
data, _ := db.Get(snapshotRootKey)
if len(data) != common.HashLength {
return common.Hash{}
}
return common.BytesToHash(data)
}
// WriteSnapshotRoot stores the root of the block whose state is contained in
// the persisted snapshot.
func WriteSnapshotRoot(db ethdb.KeyValueWriter, root common.Hash) {
if err := db.Put(snapshotRootKey, root[:]); err != nil {
log.Crit("Failed to store snapshot root", "err", err)
}
}
// DeleteSnapshotRoot deletes the hash of the block whose state is contained in
// the persisted snapshot. Since snapshots are not immutable, this method can
// be used during updates, so a crash or failure will mark the entire snapshot
// invalid.
func DeleteSnapshotRoot(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotRootKey); err != nil {
log.Crit("Failed to remove snapshot root", "err", err)
}
}
// ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf.
func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(accountSnapshotKey(hash))
return data
}
// WriteAccountSnapshot stores the snapshot entry of an account trie leaf.
func WriteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash, entry []byte) {
if err := db.Put(accountSnapshotKey(hash), entry); err != nil {
log.Crit("Failed to store account snapshot", "err", err)
}
}
// DeleteAccountSnapshot removes the snapshot entry of an account trie leaf.
func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(accountSnapshotKey(hash)); err != nil {
log.Crit("Failed to delete account snapshot", "err", err)
}
}
// ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf.
func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte {
data, _ := db.Get(storageSnapshotKey(accountHash, storageHash))
return data
}
// WriteStorageSnapshot stores the snapshot entry of an storage trie leaf.
func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) {
if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil {
log.Crit("Failed to store storage snapshot", "err", err)
}
}
// DeleteStorageSnapshot removes the snapshot entry of an storage trie leaf.
func DeleteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash) {
if err := db.Delete(storageSnapshotKey(accountHash, storageHash)); err != nil {
log.Crit("Failed to delete storage snapshot", "err", err)
}
}
// IterateStorageSnapshots returns an iterator for walking the entire storage
// space of a specific account.
func IterateStorageSnapshots(db ethdb.Iteratee, accountHash common.Hash) ethdb.Iterator {
return db.NewIterator(storageSnapshotsKey(accountHash), nil)
}
// ReadSnapshotJournal retrieves the serialized in-memory diff layers saved at
// the last shutdown. The blob is expected to be max a few 10s of megabytes.
func ReadSnapshotJournal(db ethdb.KeyValueReader) []byte {
data, _ := db.Get(snapshotJournalKey)
return data
}
// WriteSnapshotJournal stores the serialized in-memory diff layers to save at
// shutdown. The blob is expected to be max a few 10s of megabytes.
func WriteSnapshotJournal(db ethdb.KeyValueWriter, journal []byte) {
if err := db.Put(snapshotJournalKey, journal); err != nil {
log.Crit("Failed to store snapshot journal", "err", err)
}
}
// DeleteSnapshotJournal deletes the serialized in-memory diff layers saved at
// the last shutdown
func DeleteSnapshotJournal(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotJournalKey); err != nil {
log.Crit("Failed to remove snapshot journal", "err", err)
}
}

10
core/rawdb/database.go
View file

@ -221,7 +221,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database) error {
it := db.NewIterator()
it := db.NewIterator(nil, nil)
defer it.Release()
var (
@ -239,6 +239,8 @@ func InspectDatabase(db ethdb.Database) error {
hashNumPairing common.StorageSize
trieSize common.StorageSize
txlookupSize common.StorageSize
accountSnapSize common.StorageSize
storageSnapSize common.StorageSize
preimageSize common.StorageSize
bloomBitsSize common.StorageSize
cliqueSnapsSize common.StorageSize
@ -280,6 +282,10 @@ func InspectDatabase(db ethdb.Database) error {
receiptSize += size
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
txlookupSize += size
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
accountSnapSize += size
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
storageSnapSize += size
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
preimageSize += size
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
@ -331,6 +337,8 @@ func InspectDatabase(db ethdb.Database) error {
{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
{"Key-Value store", "Trie nodes", trieSize.String()},
{"Key-Value store", "Trie preimages", preimageSize.String()},
{"Key-Value store", "Account snapshot", accountSnapSize.String()},
{"Key-Value store", "Storage snapshot", storageSnapSize.String()},
{"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()},
{"Key-Value store", "Singleton metadata", metadata.String()},
{"Ancient store", "Headers", ancientHeaders.String()},

10
core/rawdb/freezer_table.go
View file

@ -541,20 +541,22 @@ func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) {
// Retrieve looks up the data offset of an item with the given number and retrieves
// the raw binary blob from the data file.
func (t *freezerTable) Retrieve(item uint64) ([]byte, error) {
t.lock.RLock()
// Ensure the table and the item is accessible
if t.index == nil || t.head == nil {
t.lock.RUnlock()
return nil, errClosed
}
if atomic.LoadUint64(&t.items) <= item {
t.lock.RUnlock()
return nil, errOutOfBounds
}
// Ensure the item was not deleted from the tail either
offset := atomic.LoadUint32(&t.itemOffset)
if uint64(offset) > item {
if uint64(t.itemOffset) > item {
t.lock.RUnlock()
return nil, errOutOfBounds
}
t.lock.RLock()
startOffset, endOffset, filenum, err := t.getBounds(item - uint64(offset))
startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset))
if err != nil {
t.lock.RUnlock()
return nil, err

6
core/rawdb/freezer_table_test.go
View file

@ -196,10 +196,8 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
f.Append(uint64(x), data)
}
// The last item should be there
if _, err = f.Retrieve(f.items - 1); err == nil {
if err != nil {
t.Fatal(err)
}
if _, err = f.Retrieve(f.items - 1); err != nil {
t.Fatal(err)
}
f.Close()
}

27
core/rawdb/schema.go
View file

@ -41,6 +41,12 @@ var (
// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
fastTrieProgressKey = []byte("TrieSync")
// snapshotRootKey tracks the hash of the last snapshot.
snapshotRootKey = []byte("SnapshotRoot")
// snapshotJournalKey tracks the in-memory diff layers across restarts.
snapshotJournalKey = []byte("SnapshotJournal")
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
@ -50,8 +56,10 @@ var (
blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body
blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts
txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value
SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value
preimagePrefix = []byte("secure-key-") // preimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
@ -145,6 +153,21 @@ func txLookupKey(hash common.Hash) []byte {
return append(txLookupPrefix, hash.Bytes()...)
}
// accountSnapshotKey = SnapshotAccountPrefix + hash
func accountSnapshotKey(hash common.Hash) []byte {
return append(SnapshotAccountPrefix, hash.Bytes()...)
}
// storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash
func storageSnapshotKey(accountHash, storageHash common.Hash) []byte {
return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...)
}
// storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash
func storageSnapshotsKey(accountHash common.Hash) []byte {
return append(SnapshotStoragePrefix, accountHash.Bytes()...)
}
// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...)

91
core/rawdb/table.go
View file

@ -103,23 +103,16 @@ func (t *table) Delete(key []byte) error {
return t.db.Delete(append([]byte(t.prefix), key...))
}
// NewIterator creates a binary-alphabetical iterator over the entire keyspace
// contained within the database.
func (t *table) NewIterator() ethdb.Iterator {
return t.NewIteratorWithPrefix(nil)
}
// NewIteratorWithStart creates a binary-alphabetical iterator over a subset of
// database content starting at a particular initial key (or after, if it does
// not exist).
func (t *table) NewIteratorWithStart(start []byte) ethdb.Iterator {
return t.db.NewIteratorWithStart(start)
}
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix.
func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
return t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...))
// NewIterator creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix, starting at a particular
// initial key (or after, if it does not exist).
func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
innerPrefix := append([]byte(t.prefix), prefix...)
iter := t.db.NewIterator(innerPrefix, start)
return &tableIterator{
iter: iter,
prefix: t.prefix,
}
}
// Stat returns a particular internal stat of the database.
@ -198,7 +191,69 @@ func (b *tableBatch) Reset() {
b.batch.Reset()
}
// tableReplayer is a wrapper around a batch replayer which truncates
// the added prefix.
type tableReplayer struct {
w ethdb.KeyValueWriter
prefix string
}
// Put implements the interface KeyValueWriter.
func (r *tableReplayer) Put(key []byte, value []byte) error {
trimmed := key[len(r.prefix):]
return r.w.Put(trimmed, value)
}
// Delete implements the interface KeyValueWriter.
func (r *tableReplayer) Delete(key []byte) error {
trimmed := key[len(r.prefix):]
return r.w.Delete(trimmed)
}
// Replay replays the batch contents.
func (b *tableBatch) Replay(w ethdb.KeyValueWriter) error {
return b.batch.Replay(w)
return b.batch.Replay(&tableReplayer{w: w, prefix: b.prefix})
}
// tableIterator is a wrapper around a database iterator that prefixes each key access
// with a pre-configured string.
type tableIterator struct {
iter ethdb.Iterator
prefix string
}
// Next moves the iterator to the next key/value pair. It returns whether the
// iterator is exhausted.
func (iter *tableIterator) Next() bool {
return iter.iter.Next()
}
// Error returns any accumulated error. Exhausting all the key/value pairs
// is not considered to be an error.
func (iter *tableIterator) Error() error {
return iter.iter.Error()
}
// Key returns the key of the current key/value pair, or nil if done. The caller
// should not modify the contents of the returned slice, and its contents may
// change on the next call to Next.
func (iter *tableIterator) Key() []byte {
key := iter.iter.Key()
if key == nil {
return nil
}
return key[len(iter.prefix):]
}
// Value returns the value of the current key/value pair, or nil if done. The
// caller should not modify the contents of the returned slice, and its contents
// may change on the next call to Next.
func (iter *tableIterator) Value() []byte {
return iter.iter.Value()
}
// Release releases associated resources. Release should always succeed and can
// be called multiple times without causing error.
func (iter *tableIterator) Release() {
iter.iter.Release()
}

128
core/rawdb/table_test.go Normal file
View file

@ -0,0 +1,128 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"bytes"
"testing"
"github.com/ethereum/go-ethereum/ethdb"
)
func TestTableDatabase(t *testing.T) { testTableDatabase(t, "prefix") }
func TestEmptyPrefixTableDatabase(t *testing.T) { testTableDatabase(t, "") }
type testReplayer struct {
puts [][]byte
dels [][]byte
}
func (r *testReplayer) Put(key []byte, value []byte) error {
r.puts = append(r.puts, key)
return nil
}
func (r *testReplayer) Delete(key []byte) error {
r.dels = append(r.dels, key)
return nil
}
func testTableDatabase(t *testing.T, prefix string) {
db := NewTable(NewMemoryDatabase(), prefix)
var entries = []struct {
key []byte
value []byte
}{
{[]byte{0x01, 0x02}, []byte{0x0a, 0x0b}},
{[]byte{0x03, 0x04}, []byte{0x0c, 0x0d}},
{[]byte{0x05, 0x06}, []byte{0x0e, 0x0f}},
{[]byte{0xff, 0xff, 0x01}, []byte{0x1a, 0x1b}},
{[]byte{0xff, 0xff, 0x02}, []byte{0x1c, 0x1d}},
{[]byte{0xff, 0xff, 0x03}, []byte{0x1e, 0x1f}},
}
// Test Put/Get operation
for _, entry := range entries {
db.Put(entry.key, entry.value)
}
for _, entry := range entries {
got, err := db.Get(entry.key)
if err != nil {
t.Fatalf("Failed to get value: %v", err)
}
if !bytes.Equal(got, entry.value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got)
}
}
// Test batch operation
db = NewTable(NewMemoryDatabase(), prefix)
batch := db.NewBatch()
for _, entry := range entries {
batch.Put(entry.key, entry.value)
}
batch.Write()
for _, entry := range entries {
got, err := db.Get(entry.key)
if err != nil {
t.Fatalf("Failed to get value: %v", err)
}
if !bytes.Equal(got, entry.value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entry.value, got)
}
}
// Test batch replayer
r := &testReplayer{}
batch.Replay(r)
for index, entry := range entries {
got := r.puts[index]
if !bytes.Equal(got, entry.key) {
t.Fatalf("Key mismatch: want=%v, got=%v", entry.key, got)
}
}
check := func(iter ethdb.Iterator, expCount, index int) {
count := 0
for iter.Next() {
key, value := iter.Key(), iter.Value()
if !bytes.Equal(key, entries[index].key) {
t.Fatalf("Key mismatch: want=%v, got=%v", entries[index].key, key)
}
if !bytes.Equal(value, entries[index].value) {
t.Fatalf("Value mismatch: want=%v, got=%v", entries[index].value, value)
}
index += 1
count++
}
if count != expCount {
t.Fatalf("Wrong number of elems, exp %d got %d", expCount, count)
}
iter.Release()
}
// Test iterators
check(db.NewIterator(nil, nil), 6, 0)
// Test iterators with prefix
check(db.NewIterator([]byte{0xff, 0xff}, nil), 3, 3)
// Test iterators with start point
check(db.NewIterator(nil, []byte{0xff, 0xff, 0x02}), 2, 4)
// Test iterators with prefix and start point
check(db.NewIterator([]byte{0xee}, nil), 0, 0)
check(db.NewIterator(nil, []byte{0x00}), 6, 0)
}

48
core/state/dump.go
View file

@ -27,7 +27,7 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
// DumpAccount represents an account in the state
// DumpAccount represents an account in the state.
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
@ -40,17 +40,24 @@ type DumpAccount struct {
}
// Dump represents the full dump in a collected format, as one large map
// Dump represents the full dump in a collected format, as one large map.
type Dump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
type iterativeDump struct {
*json.Encoder
}
// IteratorDump is an implementation for iterating over data.
type IteratorDump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
Next []byte `json:"next,omitempty"` // nil if no more accounts
}
// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d *IteratorDump) onRoot(root common.Hash) {
d.Root = fmt.Sprintf("%x", root)
}
func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
}{root})
}
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(s.trie.Hash())
it := trie.NewIterator(s.trie.NodeIterator(nil))
var count int
it := trie.NewIterator(s.trie.NodeIterator(start))
for it.Next() {
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
@ -130,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
}
}
c.onAccount(addr, account)
count++
if maxResults > 0 && count >= maxResults {
if it.Next() {
nextKey = it.Key
}
break
}
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
return nextKey
}
// RawDump returns the entire state an a single large object
@ -141,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
return *dump
}
@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
}
// IteratorDump dumps out a batch of accounts starts with the given start key
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
iterator := &IteratorDump{
Accounts: make(map[common.Address]DumpAccount),
}
iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
return *iterator
}

4
core/state/iterator_test.go
View file

@ -29,7 +29,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
// Create some arbitrary test state to iterate
db, root, _ := makeTestState()
state, err := New(root, db)
state, err := New(root, db, nil)
if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err)
}
@ -51,7 +51,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
t.Errorf("state entry not reported %x", hash)
}
}
it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator()
it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator(nil, nil)
for it.Next() {
key := it.Key()
if bytes.HasPrefix(key, []byte("secure-key-")) {

6
core/state/journal.go
View file

@ -90,7 +90,8 @@ type (
account *common.Address
}
resetObjectChange struct {
prev *stateObject
prev *stateObject
prevdestruct bool
}
suicideChange struct {
account *common.Address
@ -142,6 +143,9 @@ func (ch createObjectChange) dirtied() *common.Address {
func (ch resetObjectChange) revert(s *StateDB) {
s.setStateObject(ch.prev)
if !ch.prevdestruct && s.snap != nil {
delete(s.snapDestructs, ch.prev.addrHash)
}
}
func (ch resetObjectChange) dirtied() *common.Address {

54
core/state/snapshot/account.go Normal file
View file

@ -0,0 +1,54 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
)
// Account is a slim version of a state.Account, where the root and code hash
// are replaced with a nil byte slice for empty accounts.
type Account struct {
Nonce uint64
Balance *big.Int
Root []byte
CodeHash []byte
}
// AccountRLP converts a state.Account content into a slim snapshot version RLP
// encoded.
func AccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byte) []byte {
slim := Account{
Nonce: nonce,
Balance: balance,
}
if root != emptyRoot {
slim.Root = root[:]
}
if !bytes.Equal(codehash, emptyCode[:]) {
slim.CodeHash = codehash
}
data, err := rlp.EncodeToBytes(slim)
if err != nil {
panic(err)
}
return data
}

114
core/state/snapshot/conversion.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// conversionAccount is used for converting between full and slim format. When
// doing this, we can consider 'balance' as a byte array, as it has already
// been converted from big.Int into an rlp-byteslice.
type conversionAccount struct {
Nonce uint64
Balance []byte
Root []byte
CodeHash []byte
}
// SlimToFull converts data on the 'slim RLP' format into the full RLP-format
func SlimToFull(data []byte) ([]byte, error) {
acc := &conversionAccount{}
if err := rlp.DecodeBytes(data, acc); err != nil {
return nil, err
}
if len(acc.Root) == 0 {
acc.Root = emptyRoot[:]
}
if len(acc.CodeHash) == 0 {
acc.CodeHash = emptyCode[:]
}
fullData, err := rlp.EncodeToBytes(acc)
if err != nil {
return nil, err
}
return fullData, nil
}
// trieKV represents a trie key-value pair
type trieKV struct {
key common.Hash
value []byte
}
type trieGeneratorFn func(in chan (trieKV), out chan (common.Hash))
// GenerateTrieRoot takes an account iterator and reproduces the root hash.
func GenerateTrieRoot(it AccountIterator) common.Hash {
return generateTrieRoot(it, stdGenerate)
}
func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash {
var (
in = make(chan trieKV) // chan to pass leaves
out = make(chan common.Hash) // chan to collect result
wg sync.WaitGroup
)
wg.Add(1)
go func() {
generatorFn(in, out)
wg.Done()
}()
// Feed leaves
start := time.Now()
logged := time.Now()
accounts := 0
for it.Next() {
slimData := it.Account()
fullData, _ := SlimToFull(slimData)
l := trieKV{it.Hash(), fullData}
in <- l
if time.Since(logged) > 8*time.Second {
log.Info("Generating trie hash from snapshot",
"at", l.key, "accounts", accounts, "elapsed", time.Since(start))
logged = time.Now()
}
accounts++
}
close(in)
result := <-out
log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start))
wg.Wait()
return result
}
// stdGenerate is a very basic hexary trie builder which uses the same Trie
// as the rest of geth, with no enhancements or optimizations
func stdGenerate(in chan (trieKV), out chan (common.Hash)) {
t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New()))
for leaf := range in {
t.TryUpdate(leaf.key[:], leaf.value)
}
out <- t.Hash()
}

533
core/state/snapshot/difflayer.go Normal file
View file

@ -0,0 +1,533 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"encoding/binary"
"fmt"
"math"
"math/rand"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
"github.com/steakknife/bloomfilter"
)
var (
// aggregatorMemoryLimit is the maximum size of the bottom-most diff layer
// that aggregates the writes from above until it's flushed into the disk
// layer.
//
// Note, bumping this up might drastically increase the size of the bloom
// filters that's stored in every diff layer. Don't do that without fully
// understanding all the implications.
aggregatorMemoryLimit = uint64(4 * 1024 * 1024)
// aggregatorItemLimit is an approximate number of items that will end up
// in the agregator layer before it's flushed out to disk. A plain account
// weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot
// 0B (+hash). Slots are mostly set/unset in lockstep, so thet average at
// 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a
// smaller number to be on the safe side.
aggregatorItemLimit = aggregatorMemoryLimit / 42
// bloomTargetError is the target false positive rate when the aggregator
// layer is at its fullest. The actual value will probably move around up
// and down from this number, it's mostly a ballpark figure.
//
// Note, dropping this down might drastically increase the size of the bloom
// filters that's stored in every diff layer. Don't do that without fully
// understanding all the implications.
bloomTargetError = 0.02
// bloomSize is the ideal bloom filter size given the maximum number of items
// it's expected to hold and the target false positive error rate.
bloomSize = math.Ceil(float64(aggregatorItemLimit) * math.Log(bloomTargetError) / math.Log(1/math.Pow(2, math.Log(2))))
// bloomFuncs is the ideal number of bits a single entry should set in the
// bloom filter to keep its size to a minimum (given it's size and maximum
// entry count).
bloomFuncs = math.Round((bloomSize / float64(aggregatorItemLimit)) * math.Log(2))
// the bloom offsets are runtime constants which determines which part of the
// the account/storage hash the hasher functions looks at, to determine the
// bloom key for an account/slot. This is randomized at init(), so that the
// global population of nodes do not all display the exact same behaviour with
// regards to bloom content
bloomDestructHasherOffset = 0
bloomAccountHasherOffset = 0
bloomStorageHasherOffset = 0
)
func init() {
// Init the bloom offsets in the range [0:24] (requires 8 bytes)
bloomDestructHasherOffset = rand.Intn(25)
bloomAccountHasherOffset = rand.Intn(25)
bloomStorageHasherOffset = rand.Intn(25)
// The destruct and account blooms must be different, as the storage slots
// will check for destruction too for every bloom miss. It should not collide
// with modified accounts.
for bloomAccountHasherOffset == bloomDestructHasherOffset {
bloomAccountHasherOffset = rand.Intn(25)
}
}
// diffLayer represents a collection of modifications made to a state snapshot
// after running a block on top. It contains one sorted list for the account trie
// and one-one list for each storage tries.
//
// The goal of a diff layer is to act as a journal, tracking recent modifications
// made to the state, that have not yet graduated into a semi-immutable state.
type diffLayer struct {
origin *diskLayer // Base disk layer to directly use on bloom misses
parent snapshot // Parent snapshot modified by this one, never nil
memory uint64 // Approximate guess as to how much memory we use
root common.Hash // Root hash to which this snapshot diff belongs to
stale uint32 // Signals that the layer became stale (state progressed)
destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts
accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil
accountData map[common.Hash][]byte // Keyed accounts for direct retrival (nil means deleted)
storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrival. one per account (nil means deleted)
diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
lock sync.RWMutex
}
// destructBloomHasher is a wrapper around a common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert a destruct
// event into a 64 bit mini hash.
type destructBloomHasher common.Hash
func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h destructBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h destructBloomHasher) Reset() { panic("not implemented") }
func (h destructBloomHasher) BlockSize() int { panic("not implemented") }
func (h destructBloomHasher) Size() int { return 8 }
func (h destructBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8])
}
// accountBloomHasher is a wrapper around a common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert an account
// hash into a 64 bit mini hash.
type accountBloomHasher common.Hash
func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h accountBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h accountBloomHasher) Reset() { panic("not implemented") }
func (h accountBloomHasher) BlockSize() int { panic("not implemented") }
func (h accountBloomHasher) Size() int { return 8 }
func (h accountBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8])
}
// storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface
// API requirements of the bloom library used. It's used to convert an account
// hash into a 64 bit mini hash.
type storageBloomHasher [2]common.Hash
func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (h storageBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (h storageBloomHasher) Reset() { panic("not implemented") }
func (h storageBloomHasher) BlockSize() int { panic("not implemented") }
func (h storageBloomHasher) Size() int { return 8 }
func (h storageBloomHasher) Sum64() uint64 {
return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^
binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8])
}
// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
// level persistent database or a hierarchical diff already.
func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
// Create the new layer with some pre-allocated data segments
dl := &diffLayer{
parent: parent,
root: root,
destructSet: destructs,
accountData: accounts,
storageData: storage,
}
switch parent := parent.(type) {
case *diskLayer:
dl.rebloom(parent)
case *diffLayer:
dl.rebloom(parent.origin)
default:
panic("unknown parent type")
}
// Sanity check that accounts or storage slots are never nil
for accountHash, blob := range accounts {
if blob == nil {
panic(fmt.Sprintf("account %#x nil", accountHash))
}
}
for accountHash, slots := range storage {
if slots == nil {
panic(fmt.Sprintf("storage %#x nil", accountHash))
}
}
// Determine memory size and track the dirty writes
for _, data := range accounts {
dl.memory += uint64(common.HashLength + len(data))
snapshotDirtyAccountWriteMeter.Mark(int64(len(data)))
}
// Fill the storage hashes and sort them for the iterator
dl.storageList = make(map[common.Hash][]common.Hash)
for accountHash := range destructs {
dl.storageList[accountHash] = nil
}
// Determine memory size and track the dirty writes
for _, slots := range storage {
for _, data := range slots {
dl.memory += uint64(common.HashLength + len(data))
snapshotDirtyStorageWriteMeter.Mark(int64(len(data)))
}
}
dl.memory += uint64(len(dl.storageList) * common.HashLength)
return dl
}
// rebloom discards the layer's current bloom and rebuilds it from scratch based
// on the parent's and the local diffs.
func (dl *diffLayer) rebloom(origin *diskLayer) {
dl.lock.Lock()
defer dl.lock.Unlock()
defer func(start time.Time) {
snapshotBloomIndexTimer.Update(time.Since(start))
}(time.Now())
// Inject the new origin that triggered the rebloom
dl.origin = origin
// Retrieve the parent bloom or create a fresh empty one
if parent, ok := dl.parent.(*diffLayer); ok {
parent.lock.RLock()
dl.diffed, _ = parent.diffed.Copy()
parent.lock.RUnlock()
} else {
dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
}
// Iterate over all the accounts and storage slots and index them
for hash := range dl.destructSet {
dl.diffed.Add(destructBloomHasher(hash))
}
for hash := range dl.accountData {
dl.diffed.Add(accountBloomHasher(hash))
}
for accountHash, slots := range dl.storageData {
for storageHash := range slots {
dl.diffed.Add(storageBloomHasher{accountHash, storageHash})
}
}
// Calculate the current false positive rate and update the error rate meter.
// This is a bit cheating because subsequent layers will overwrite it, but it
// should be fine, we're only interested in ballpark figures.
k := float64(dl.diffed.K())
n := float64(dl.diffed.N())
m := float64(dl.diffed.M())
snapshotBloomErrorGauge.Update(math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k))
}
// Root returns the root hash for which this snapshot was made.
func (dl *diffLayer) Root() common.Hash {
return dl.root
}
// Parent returns the subsequent layer of a diff layer.
func (dl *diffLayer) Parent() snapshot {
return dl.parent
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diffLayer) Stale() bool {
return atomic.LoadUint32(&dl.stale) != 0
}
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
data, err := dl.AccountRLP(hash)
if err != nil {
return nil, err
}
if len(data) == 0 { // can be both nil and []byte{}
return nil, nil
}
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
return account, nil
}
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
// Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below
dl.lock.RLock()
hit := dl.diffed.Contains(accountBloomHasher(hash))
if !hit {
hit = dl.diffed.Contains(destructBloomHasher(hash))
}
dl.lock.RUnlock()
// If the bloom filter misses, don't even bother with traversing the memory
// diff layers, reach straight into the bottom persistent disk layer
if !hit {
snapshotBloomAccountMissMeter.Mark(1)
return dl.origin.AccountRLP(hash)
}
// The bloom filter hit, start poking in the internal maps
return dl.accountRLP(hash, 0)
}
// accountRLP is an internal version of AccountRLP that skips the bloom filter
// checks and uses the internal maps to try and retrieve the data. It's meant
// to be used if a higher layer's bloom filter hit already.
func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.Stale() {
return nil, ErrSnapshotStale
}
// If the account is known locally, return it
if data, ok := dl.accountData[hash]; ok {
snapshotDirtyAccountHitMeter.Mark(1)
snapshotDirtyAccountHitDepthHist.Update(int64(depth))
snapshotDirtyAccountReadMeter.Mark(int64(len(data)))
snapshotBloomAccountTrueHitMeter.Mark(1)
return data, nil
}
// If the account is known locally, but deleted, return it
if _, ok := dl.destructSet[hash]; ok {
snapshotDirtyAccountHitMeter.Mark(1)
snapshotDirtyAccountHitDepthHist.Update(int64(depth))
snapshotDirtyAccountInexMeter.Mark(1)
snapshotBloomAccountTrueHitMeter.Mark(1)
return nil, nil
}
// Account unknown to this diff, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.accountRLP(hash, depth+1)
}
// Failed to resolve through diff layers, mark a bloom error and use the disk
snapshotBloomAccountFalseHitMeter.Mark(1)
return dl.parent.AccountRLP(hash)
}
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account. If the slot is unknown to this diff, it's parent
// is consulted.
func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
// Check the bloom filter first whether there's even a point in reaching into
// all the maps in all the layers below
dl.lock.RLock()
hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash})
if !hit {
hit = dl.diffed.Contains(destructBloomHasher(accountHash))
}
dl.lock.RUnlock()
// If the bloom filter misses, don't even bother with traversing the memory
// diff layers, reach straight into the bottom persistent disk layer
if !hit {
snapshotBloomStorageMissMeter.Mark(1)
return dl.origin.Storage(accountHash, storageHash)
}
// The bloom filter hit, start poking in the internal maps
return dl.storage(accountHash, storageHash, 0)
}
// storage is an internal version of Storage that skips the bloom filter checks
// and uses the internal maps to try and retrieve the data. It's meant to be
// used if a higher layer's bloom filter hit already.
func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.Stale() {
return nil, ErrSnapshotStale
}
// If the account is known locally, try to resolve the slot locally
if storage, ok := dl.storageData[accountHash]; ok {
if data, ok := storage[storageHash]; ok {
snapshotDirtyStorageHitMeter.Mark(1)
snapshotDirtyStorageHitDepthHist.Update(int64(depth))
if n := len(data); n > 0 {
snapshotDirtyStorageReadMeter.Mark(int64(n))
} else {
snapshotDirtyStorageInexMeter.Mark(1)
}
snapshotBloomStorageTrueHitMeter.Mark(1)
return data, nil
}
}
// If the account is known locally, but deleted, return an empty slot
if _, ok := dl.destructSet[accountHash]; ok {
snapshotDirtyStorageHitMeter.Mark(1)
snapshotDirtyStorageHitDepthHist.Update(int64(depth))
snapshotDirtyStorageInexMeter.Mark(1)
snapshotBloomStorageTrueHitMeter.Mark(1)
return nil, nil
}
// Storage slot unknown to this diff, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.storage(accountHash, storageHash, depth+1)
}
// Failed to resolve through diff layers, mark a bloom error and use the disk
snapshotBloomStorageFalseHitMeter.Mark(1)
return dl.parent.Storage(accountHash, storageHash)
}
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items.
func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, blockRoot, destructs, accounts, storage)
}
// flatten pushes all data from this point downwards, flattening everything into
// a single diff at the bottom. Since usually the lowermost diff is the largest,
// the flattening bulds up from there in reverse.
func (dl *diffLayer) flatten() snapshot {
// If the parent is not diff, we're the first in line, return unmodified
parent, ok := dl.parent.(*diffLayer)
if !ok {
return dl
}
// Parent is a diff, flatten it first (note, apart from weird corned cases,
// flatten will realistically only ever merge 1 layer, so there's no need to
// be smarter about grouping flattens together).
parent = parent.flatten().(*diffLayer)
parent.lock.Lock()
defer parent.lock.Unlock()
// Before actually writing all our data to the parent, first ensure that the
// parent hasn't been 'corrupted' by someone else already flattening into it
if atomic.SwapUint32(&parent.stale, 1) != 0 {
panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo
}
// Overwrite all the updated accounts blindly, merge the sorted list
for hash := range dl.destructSet {
parent.destructSet[hash] = struct{}{}
delete(parent.accountData, hash)
delete(parent.storageData, hash)
}
for hash, data := range dl.accountData {
parent.accountData[hash] = data
}
// Overwrite all the updated storage slots (individually)
for accountHash, storage := range dl.storageData {
// If storage didn't exist (or was deleted) in the parent, overwrite blindly
if _, ok := parent.storageData[accountHash]; !ok {
parent.storageData[accountHash] = storage
continue
}
// Storage exists in both parent and child, merge the slots
comboData := parent.storageData[accountHash]
for storageHash, data := range storage {
comboData[storageHash] = data
}
parent.storageData[accountHash] = comboData
}
// Return the combo parent
return &diffLayer{
parent: parent.parent,
origin: parent.origin,
root: dl.root,
destructSet: parent.destructSet,
accountData: parent.accountData,
storageData: parent.storageData,
storageList: make(map[common.Hash][]common.Hash),
diffed: dl.diffed,
memory: parent.memory + dl.memory,
}
}
// AccountList returns a sorted list of all accounts in this difflayer, including
// the deleted ones.
//
// Note, the returned slice is not a copy, so do not modify it.
func (dl *diffLayer) AccountList() []common.Hash {
// If an old list already exists, return it
dl.lock.RLock()
list := dl.accountList
dl.lock.RUnlock()
if list != nil {
return list
}
// No old sorted account list exists, generate a new one
dl.lock.Lock()
defer dl.lock.Unlock()
dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData))
for hash := range dl.accountData {
dl.accountList = append(dl.accountList, hash)
}
for hash := range dl.destructSet {
if _, ok := dl.accountData[hash]; !ok {
dl.accountList = append(dl.accountList, hash)
}
}
sort.Sort(hashes(dl.accountList))
return dl.accountList
}
// StorageList returns a sorted list of all storage slot hashes in this difflayer
// for the given account.
//
// Note, the returned slice is not a copy, so do not modify it.
func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash {
// If an old list already exists, return it
dl.lock.RLock()
list := dl.storageList[accountHash]
dl.lock.RUnlock()
if list != nil {
return list
}
// No old sorted account list exists, generate a new one
dl.lock.Lock()
defer dl.lock.Unlock()
storageMap := dl.storageData[accountHash]
storageList := make([]common.Hash, 0, len(storageMap))
for k := range storageMap {
storageList = append(storageList, k)
}
sort.Sort(hashes(storageList))
dl.storageList[accountHash] = storageList
return storageList
}

399
core/state/snapshot/difflayer_test.go Normal file
View file

@ -0,0 +1,399 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"math/rand"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} {
copy := make(map[common.Hash]struct{})
for hash := range destructs {
copy[hash] = struct{}{}
}
return copy
}
func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte {
copy := make(map[common.Hash][]byte)
for hash, blob := range accounts {
copy[hash] = blob
}
return copy
}
func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte {
copy := make(map[common.Hash]map[common.Hash][]byte)
for accHash, slots := range storage {
copy[accHash] = make(map[common.Hash][]byte)
for slotHash, blob := range slots {
copy[accHash][slotHash] = blob
}
}
return copy
}
// TestMergeBasics tests some simple merges
func TestMergeBasics(t *testing.T) {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
// Fill up a parent
for i := 0; i < 100; i++ {
h := randomHash()
data := randomAccount()
accounts[h] = data
if rand.Intn(4) == 0 {
destructs[h] = struct{}{}
}
if rand.Intn(2) == 0 {
accStorage := make(map[common.Hash][]byte)
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
storage[h] = accStorage
}
}
// Add some (identical) layers on top
parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
// And flatten
merged := (child.flatten()).(*diffLayer)
{ // Check account lists
if have, want := len(merged.accountList), 0; have != want {
t.Errorf("accountList wrong: have %v, want %v", have, want)
}
if have, want := len(merged.AccountList()), len(accounts); have != want {
t.Errorf("AccountList() wrong: have %v, want %v", have, want)
}
if have, want := len(merged.accountList), len(accounts); have != want {
t.Errorf("accountList [2] wrong: have %v, want %v", have, want)
}
}
{ // Check account drops
if have, want := len(merged.destructSet), len(destructs); have != want {
t.Errorf("accountDrop wrong: have %v, want %v", have, want)
}
}
{ // Check storage lists
i := 0
for aHash, sMap := range storage {
if have, want := len(merged.storageList), i; have != want {
t.Errorf("[1] storageList wrong: have %v, want %v", have, want)
}
if have, want := len(merged.StorageList(aHash)), len(sMap); have != want {
t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want)
}
if have, want := len(merged.storageList[aHash]), len(sMap); have != want {
t.Errorf("storageList wrong: have %v, want %v", have, want)
}
i++
}
}
}
// TestMergeDelete tests some deletion
func TestMergeDelete(t *testing.T) {
var (
storage = make(map[common.Hash]map[common.Hash][]byte)
)
// Fill up a parent
h1 := common.HexToHash("0x01")
h2 := common.HexToHash("0x02")
flipDrops := func() map[common.Hash]struct{} {
return map[common.Hash]struct{}{
h2: struct{}{},
}
}
flipAccs := func() map[common.Hash][]byte {
return map[common.Hash][]byte{
h1: randomAccount(),
}
}
flopDrops := func() map[common.Hash]struct{} {
return map[common.Hash]struct{}{
h1: struct{}{},
}
}
flopAccs := func() map[common.Hash][]byte {
return map[common.Hash][]byte{
h2: randomAccount(),
}
}
// Add some flipAccs-flopping layers on top
parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage)
child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
if data, _ := child.Account(h1); data == nil {
t.Errorf("last diff layer: expected %x account to be non-nil", h1)
}
if data, _ := child.Account(h2); data != nil {
t.Errorf("last diff layer: expected %x account to be nil", h2)
}
if _, ok := child.destructSet[h1]; ok {
t.Errorf("last diff layer: expected %x drop to be missing", h1)
}
if _, ok := child.destructSet[h2]; !ok {
t.Errorf("last diff layer: expected %x drop to be present", h1)
}
// And flatten
merged := (child.flatten()).(*diffLayer)
if data, _ := merged.Account(h1); data == nil {
t.Errorf("merged layer: expected %x account to be non-nil", h1)
}
if data, _ := merged.Account(h2); data != nil {
t.Errorf("merged layer: expected %x account to be nil", h2)
}
if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk!
t.Errorf("merged diff layer: expected %x drop to be present", h1)
}
if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk!
t.Errorf("merged diff layer: expected %x drop to be present", h1)
}
// If we add more granular metering of memory, we can enable this again,
// but it's not implemented for now
//if have, want := merged.memory, child.memory; have != want {
// t.Errorf("mem wrong: have %d, want %d", have, want)
//}
}
// This tests that if we create a new account, and set a slot, and then merge
// it, the lists will be correct.
func TestInsertAndMerge(t *testing.T) {
// Fill up a parent
var (
acc = common.HexToHash("0x01")
slot = common.HexToHash("0x02")
parent *diffLayer
child *diffLayer
)
{
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage)
}
{
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
accounts[acc] = randomAccount()
storage[acc] = make(map[common.Hash][]byte)
storage[acc][slot] = []byte{0x01}
child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
// And flatten
merged := (child.flatten()).(*diffLayer)
{ // Check that slot value is present
have, _ := merged.Storage(acc, slot)
if want := []byte{0x01}; !bytes.Equal(have, want) {
t.Errorf("merged slot value wrong: have %x, want %x", have, want)
}
}
}
func emptyLayer() *diskLayer {
return &diskLayer{
diskdb: memorydb.New(),
cache: fastcache.New(500 * 1024),
}
}
// BenchmarkSearch checks how long it takes to find a non-existing key
// BenchmarkSearch-6 200000 10481 ns/op (1K per layer)
// BenchmarkSearch-6 200000 10760 ns/op (10K per layer)
// BenchmarkSearch-6 100000 17866 ns/op
//
// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock()
func BenchmarkSearch(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 10000; i++ {
accounts[randomHash()] = randomAccount()
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
var layer snapshot
layer = emptyLayer()
for i := 0; i < 128; i++ {
layer = fill(layer)
}
key := crypto.Keccak256Hash([]byte{0x13, 0x38})
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.AccountRLP(key)
}
}
// BenchmarkSearchSlot checks how long it takes to find a non-existing key
// - Number of layers: 128
// - Each layers contains the account, with a couple of storage slots
// BenchmarkSearchSlot-6 100000 14554 ns/op
// BenchmarkSearchSlot-6 100000 22254 ns/op (when checking parent root using mutex)
// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic)
// With bloom filter:
// BenchmarkSearchSlot-6 3467835 351 ns/op
func BenchmarkSearchSlot(b *testing.B) {
// First, we set up 128 diff layers, with 1K items each
accountKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
storageKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
accountRLP := randomAccount()
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
accounts[accountKey] = accountRLP
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 5; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
var layer snapshot
layer = emptyLayer()
for i := 0; i < 128; i++ {
layer = fill(layer)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.Storage(accountKey, storageKey)
}
}
// With accountList and sorting
// BenchmarkFlatten-6 50 29890856 ns/op
//
// Without sorting and tracking accountlist
// BenchmarkFlatten-6 300 5511511 ns/op
func BenchmarkFlatten(b *testing.B) {
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 100; i++ {
accountKey := randomHash()
accounts[accountKey] = randomAccount()
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 20; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
}
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
var layer snapshot
layer = emptyLayer()
for i := 1; i < 128; i++ {
layer = fill(layer)
}
b.StartTimer()
for i := 1; i < 128; i++ {
dl, ok := layer.(*diffLayer)
if !ok {
break
}
layer = dl.flatten()
}
b.StopTimer()
}
}
// This test writes ~324M of diff layers to disk, spread over
// - 128 individual layers,
// - each with 200 accounts
// - containing 200 slots
//
// BenchmarkJournal-6 1 1471373923 ns/ops
// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer
func BenchmarkJournal(b *testing.B) {
fill := func(parent snapshot) *diffLayer {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
for i := 0; i < 200; i++ {
accountKey := randomHash()
accounts[accountKey] = randomAccount()
accStorage := make(map[common.Hash][]byte)
for i := 0; i < 200; i++ {
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
}
storage[accountKey] = accStorage
}
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
}
layer := snapshot(new(diskLayer))
for i := 1; i < 128; i++ {
layer = fill(layer)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
layer.Journal(new(bytes.Buffer))
}
}

166
core/state/snapshot/disklayer.go Normal file
View file

@ -0,0 +1,166 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"sync"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// diskLayer is a low level persistent snapshot built on top of a key-value store.
type diskLayer struct {
diskdb ethdb.KeyValueStore // Key-value store containing the base snapshot
triedb *trie.Database // Trie node cache for reconstuction purposes
cache *fastcache.Cache // Cache to avoid hitting the disk for direct access
root common.Hash // Root hash of the base snapshot
stale bool // Signals that the layer became stale (state progressed)
genMarker []byte // Marker for the state that's indexed during initial layer generation
genPending chan struct{} // Notification channel when generation is done (test synchronicity)
genAbort chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer
lock sync.RWMutex
}
// Root returns root hash for which this snapshot was made.
func (dl *diskLayer) Root() common.Hash {
return dl.root
}
// Parent always returns nil as there's no layer below the disk.
func (dl *diskLayer) Parent() snapshot {
return nil
}
// Stale return whether this layer has become stale (was flattened across) or if
// it's still live.
func (dl *diskLayer) Stale() bool {
dl.lock.RLock()
defer dl.lock.RUnlock()
return dl.stale
}
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
func (dl *diskLayer) Account(hash common.Hash) (*Account, error) {
data, err := dl.AccountRLP(hash)
if err != nil {
return nil, err
}
if len(data) == 0 { // can be both nil and []byte{}
return nil, nil
}
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
return account, nil
}
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
// If the layer is being generated, ensure the requested hash has already been
// covered by the generator.
if dl.genMarker != nil && bytes.Compare(hash[:], dl.genMarker) > 0 {
return nil, ErrNotCoveredYet
}
// If we're in the disk layer, all diff layers missed
snapshotDirtyAccountMissMeter.Mark(1)
// Try to retrieve the account from the memory cache
if blob, found := dl.cache.HasGet(nil, hash[:]); found {
snapshotCleanAccountHitMeter.Mark(1)
snapshotCleanAccountReadMeter.Mark(int64(len(blob)))
return blob, nil
}
// Cache doesn't contain account, pull from disk and cache for later
blob := rawdb.ReadAccountSnapshot(dl.diskdb, hash)
dl.cache.Set(hash[:], blob)
snapshotCleanAccountMissMeter.Mark(1)
if n := len(blob); n > 0 {
snapshotCleanAccountWriteMeter.Mark(int64(n))
} else {
snapshotCleanAccountInexMeter.Mark(1)
}
return blob, nil
}
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account.
func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
// If the layer was flattened into, consider it invalid (any live reference to
// the original should be marked as unusable).
if dl.stale {
return nil, ErrSnapshotStale
}
key := append(accountHash[:], storageHash[:]...)
// If the layer is being generated, ensure the requested hash has already been
// covered by the generator.
if dl.genMarker != nil && bytes.Compare(key, dl.genMarker) > 0 {
return nil, ErrNotCoveredYet
}
// If we're in the disk layer, all diff layers missed
snapshotDirtyStorageMissMeter.Mark(1)
// Try to retrieve the storage slot from the memory cache
if blob, found := dl.cache.HasGet(nil, key); found {
snapshotCleanStorageHitMeter.Mark(1)
snapshotCleanStorageReadMeter.Mark(int64(len(blob)))
return blob, nil
}
// Cache doesn't contain storage slot, pull from disk and cache for later
blob := rawdb.ReadStorageSnapshot(dl.diskdb, accountHash, storageHash)
dl.cache.Set(key, blob)
snapshotCleanStorageMissMeter.Mark(1)
if n := len(blob); n > 0 {
snapshotCleanStorageWriteMeter.Mark(int64(n))
} else {
snapshotCleanStorageInexMeter.Mark(1)
}
return blob, nil
}
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items. Note, the maps are retained by the method to avoid
// copying everything.
func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
return newDiffLayer(dl, blockHash, destructs, accounts, storage)
}

511
core/state/snapshot/disklayer_test.go Normal file
View file

@ -0,0 +1,511 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
// reverse reverses the contents of a byte slice. It's used to update random accs
// with deterministic changes.
func reverse(blob []byte) []byte {
res := make([]byte, len(blob))
for i, b := range blob {
res[len(blob)-1-i] = b
}
return res
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values.
func TestDiskMerge(t *testing.T) {
// Create some accounts in the disk layer
db := memorydb.New()
var (
accNoModNoCache = common.Hash{0x1}
accNoModCache = common.Hash{0x2}
accModNoCache = common.Hash{0x3}
accModCache = common.Hash{0x4}
accDelNoCache = common.Hash{0x5}
accDelCache = common.Hash{0x6}
conNoModNoCache = common.Hash{0x7}
conNoModNoCacheSlot = common.Hash{0x70}
conNoModCache = common.Hash{0x8}
conNoModCacheSlot = common.Hash{0x80}
conModNoCache = common.Hash{0x9}
conModNoCacheSlot = common.Hash{0x90}
conModCache = common.Hash{0xa}
conModCacheSlot = common.Hash{0xa0}
conDelNoCache = common.Hash{0xb}
conDelNoCacheSlot = common.Hash{0xb0}
conDelCache = common.Hash{0xc}
conDelCacheSlot = common.Hash{0xc0}
conNukeNoCache = common.Hash{0xd}
conNukeNoCacheSlot = common.Hash{0xd0}
conNukeCache = common.Hash{0xe}
conNukeCacheSlot = common.Hash{0xe0}
baseRoot = randomHash()
diffRoot = randomHash()
)
rawdb.WriteAccountSnapshot(db, accNoModNoCache, accNoModNoCache[:])
rawdb.WriteAccountSnapshot(db, accNoModCache, accNoModCache[:])
rawdb.WriteAccountSnapshot(db, accModNoCache, accModNoCache[:])
rawdb.WriteAccountSnapshot(db, accModCache, accModCache[:])
rawdb.WriteAccountSnapshot(db, accDelNoCache, accDelNoCache[:])
rawdb.WriteAccountSnapshot(db, accDelCache, accDelCache[:])
rawdb.WriteAccountSnapshot(db, conNoModNoCache, conNoModNoCache[:])
rawdb.WriteStorageSnapshot(db, conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNoModCache, conNoModCache[:])
rawdb.WriteStorageSnapshot(db, conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conModNoCache, conModNoCache[:])
rawdb.WriteStorageSnapshot(db, conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conModCache, conModCache[:])
rawdb.WriteStorageSnapshot(db, conModCache, conModCacheSlot, conModCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conDelNoCache, conDelNoCache[:])
rawdb.WriteStorageSnapshot(db, conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conDelCache, conDelCache[:])
rawdb.WriteStorageSnapshot(db, conDelCache, conDelCacheSlot, conDelCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNukeNoCache, conNukeNoCache[:])
rawdb.WriteStorageSnapshot(db, conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:])
rawdb.WriteAccountSnapshot(db, conNukeCache, conNukeCache[:])
rawdb.WriteStorageSnapshot(db, conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
rawdb.WriteSnapshotRoot(db, baseRoot)
// Create a disk layer based on the above and cache in some data
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
base := snaps.Snapshot(baseRoot)
base.AccountRLP(accNoModCache)
base.AccountRLP(accModCache)
base.AccountRLP(accDelCache)
base.Storage(conNoModCache, conNoModCacheSlot)
base.Storage(conModCache, conModCacheSlot)
base.Storage(conDelCache, conDelCacheSlot)
base.Storage(conNukeCache, conNukeCacheSlot)
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{
accDelNoCache: struct{}{},
accDelCache: struct{}{},
conNukeNoCache: struct{}{},
conNukeCache: struct{}{},
}, map[common.Hash][]byte{
accModNoCache: reverse(accModNoCache[:]),
accModCache: reverse(accModCache[:]),
}, map[common.Hash]map[common.Hash][]byte{
conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])},
conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])},
conDelNoCache: {conDelNoCacheSlot: nil},
conDelCache: {conDelCacheSlot: nil},
}); err != nil {
t.Fatalf("failed to update snapshot tree: %v", err)
}
if err := snaps.Cap(diffRoot, 0); err != nil {
t.Fatalf("failed to flatten snapshot tree: %v", err)
}
// Retrieve all the data through the disk layer and validate it
base = snaps.Snapshot(diffRoot)
if _, ok := base.(*diskLayer); !ok {
t.Fatalf("update not flattend into the disk layer")
}
// assertAccount ensures that an account matches the given blob.
assertAccount := func(account common.Hash, data []byte) {
t.Helper()
blob, err := base.AccountRLP(account)
if err != nil {
t.Errorf("account access (%x) failed: %v", account, err)
} else if !bytes.Equal(blob, data) {
t.Errorf("account access (%x) mismatch: have %x, want %x", account, blob, data)
}
}
assertAccount(accNoModNoCache, accNoModNoCache[:])
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModNoCache, reverse(accModNoCache[:]))
assertAccount(accModCache, reverse(accModCache[:]))
assertAccount(accDelNoCache, nil)
assertAccount(accDelCache, nil)
// assertStorage ensures that a storage slot matches the given blob.
assertStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob, err := base.Storage(account, slot)
if err != nil {
t.Errorf("storage access (%x:%x) failed: %v", account, slot, err)
} else if !bytes.Equal(blob, data) {
t.Errorf("storage access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data)
}
}
assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertStorage(conDelCache, conDelCacheSlot, nil)
assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertStorage(conNukeCache, conNukeCacheSlot, nil)
// Retrieve all the data directly from the database and validate it
// assertDatabaseAccount ensures that an account from the database matches the given blob.
assertDatabaseAccount := func(account common.Hash, data []byte) {
t.Helper()
if blob := rawdb.ReadAccountSnapshot(db, account); !bytes.Equal(blob, data) {
t.Errorf("account database access (%x) mismatch: have %x, want %x", account, blob, data)
}
}
assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:])
assertDatabaseAccount(accNoModCache, accNoModCache[:])
assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:]))
assertDatabaseAccount(accModCache, reverse(accModCache[:]))
assertDatabaseAccount(accDelNoCache, nil)
assertDatabaseAccount(accDelCache, nil)
// assertDatabaseStorage ensures that a storage slot from the database matches the given blob.
assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
if blob := rawdb.ReadStorageSnapshot(db, account, slot); !bytes.Equal(blob, data) {
t.Errorf("storage database access (%x:%x) mismatch: have %x, want %x", account, slot, blob, data)
}
}
assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertDatabaseStorage(conDelCache, conDelCacheSlot, nil)
assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil)
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values, discarding anything
// after the in-progress generation marker.
func TestDiskPartialMerge(t *testing.T) {
// Iterate the test a few times to ensure we pick various internal orderings
// for the data slots as well as the progress marker.
for i := 0; i < 1024; i++ {
// Create some accounts in the disk layer
db := memorydb.New()
var (
accNoModNoCache = randomHash()
accNoModCache = randomHash()
accModNoCache = randomHash()
accModCache = randomHash()
accDelNoCache = randomHash()
accDelCache = randomHash()
conNoModNoCache = randomHash()
conNoModNoCacheSlot = randomHash()
conNoModCache = randomHash()
conNoModCacheSlot = randomHash()
conModNoCache = randomHash()
conModNoCacheSlot = randomHash()
conModCache = randomHash()
conModCacheSlot = randomHash()
conDelNoCache = randomHash()
conDelNoCacheSlot = randomHash()
conDelCache = randomHash()
conDelCacheSlot = randomHash()
conNukeNoCache = randomHash()
conNukeNoCacheSlot = randomHash()
conNukeCache = randomHash()
conNukeCacheSlot = randomHash()
baseRoot = randomHash()
diffRoot = randomHash()
genMarker = append(randomHash().Bytes(), randomHash().Bytes()...)
)
// insertAccount injects an account into the database if it's after the
// generator marker, drops the op otherwise. This is needed to seed the
// database with a valid starting snapshot.
insertAccount := func(account common.Hash, data []byte) {
if bytes.Compare(account[:], genMarker) <= 0 {
rawdb.WriteAccountSnapshot(db, account, data[:])
}
}
insertAccount(accNoModNoCache, accNoModNoCache[:])
insertAccount(accNoModCache, accNoModCache[:])
insertAccount(accModNoCache, accModNoCache[:])
insertAccount(accModCache, accModCache[:])
insertAccount(accDelNoCache, accDelNoCache[:])
insertAccount(accDelCache, accDelCache[:])
// insertStorage injects a storage slot into the database if it's after
// the generator marker, drops the op otherwise. This is needed to seed
// the database with a valid starting snapshot.
insertStorage := func(account common.Hash, slot common.Hash, data []byte) {
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 {
rawdb.WriteStorageSnapshot(db, account, slot, data[:])
}
}
insertAccount(conNoModNoCache, conNoModNoCache[:])
insertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
insertAccount(conNoModCache, conNoModCache[:])
insertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
insertAccount(conModNoCache, conModNoCache[:])
insertStorage(conModNoCache, conModNoCacheSlot, conModNoCacheSlot[:])
insertAccount(conModCache, conModCache[:])
insertStorage(conModCache, conModCacheSlot, conModCacheSlot[:])
insertAccount(conDelNoCache, conDelNoCache[:])
insertStorage(conDelNoCache, conDelNoCacheSlot, conDelNoCacheSlot[:])
insertAccount(conDelCache, conDelCache[:])
insertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:])
insertAccount(conNukeNoCache, conNukeNoCache[:])
insertStorage(conNukeNoCache, conNukeNoCacheSlot, conNukeNoCacheSlot[:])
insertAccount(conNukeCache, conNukeCache[:])
insertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
rawdb.WriteSnapshotRoot(db, baseRoot)
// Create a disk layer based on the above using a random progress marker
// and cache in some data.
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
snaps.layers[baseRoot].(*diskLayer).genMarker = genMarker
base := snaps.Snapshot(baseRoot)
// assertAccount ensures that an account matches the given blob if it's
// already covered by the disk snapshot, and errors out otherwise.
assertAccount := func(account common.Hash, data []byte) {
t.Helper()
blob, err := base.AccountRLP(account)
if bytes.Compare(account[:], genMarker) > 0 && err != ErrNotCoveredYet {
t.Fatalf("test %d: post-marker (%x) account access (%x) succeeded: %x", i, genMarker, account, blob)
}
if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) account access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data)
}
}
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModCache, accModCache[:])
assertAccount(accDelCache, accDelCache[:])
// assertStorage ensures that a storage slot matches the given blob if
// it's already covered by the disk snapshot, and errors out otherwise.
assertStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob, err := base.Storage(account, slot)
if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && err != ErrNotCoveredYet {
t.Fatalf("test %d: post-marker (%x) storage access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob)
}
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) storage access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data)
}
}
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModCache, conModCacheSlot, conModCacheSlot[:])
assertStorage(conDelCache, conDelCacheSlot, conDelCacheSlot[:])
assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:])
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{
accDelNoCache: struct{}{},
accDelCache: struct{}{},
conNukeNoCache: struct{}{},
conNukeCache: struct{}{},
}, map[common.Hash][]byte{
accModNoCache: reverse(accModNoCache[:]),
accModCache: reverse(accModCache[:]),
}, map[common.Hash]map[common.Hash][]byte{
conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])},
conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])},
conDelNoCache: {conDelNoCacheSlot: nil},
conDelCache: {conDelCacheSlot: nil},
}); err != nil {
t.Fatalf("test %d: failed to update snapshot tree: %v", i, err)
}
if err := snaps.Cap(diffRoot, 0); err != nil {
t.Fatalf("test %d: failed to flatten snapshot tree: %v", i, err)
}
// Retrieve all the data through the disk layer and validate it
base = snaps.Snapshot(diffRoot)
if _, ok := base.(*diskLayer); !ok {
t.Fatalf("test %d: update not flattend into the disk layer", i)
}
assertAccount(accNoModNoCache, accNoModNoCache[:])
assertAccount(accNoModCache, accNoModCache[:])
assertAccount(accModNoCache, reverse(accModNoCache[:]))
assertAccount(accModCache, reverse(accModCache[:]))
assertAccount(accDelNoCache, nil)
assertAccount(accDelCache, nil)
assertStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertStorage(conDelCache, conDelCacheSlot, nil)
assertStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertStorage(conNukeCache, conNukeCacheSlot, nil)
// Retrieve all the data directly from the database and validate it
// assertDatabaseAccount ensures that an account inside the database matches
// the given blob if it's already covered by the disk snapshot, and does not
// exist otherwise.
assertDatabaseAccount := func(account common.Hash, data []byte) {
t.Helper()
blob := rawdb.ReadAccountSnapshot(db, account)
if bytes.Compare(account[:], genMarker) > 0 && blob != nil {
t.Fatalf("test %d: post-marker (%x) account database access (%x) succeeded: %x", i, genMarker, account, blob)
}
if bytes.Compare(account[:], genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) account database access (%x) mismatch: have %x, want %x", i, genMarker, account, blob, data)
}
}
assertDatabaseAccount(accNoModNoCache, accNoModNoCache[:])
assertDatabaseAccount(accNoModCache, accNoModCache[:])
assertDatabaseAccount(accModNoCache, reverse(accModNoCache[:]))
assertDatabaseAccount(accModCache, reverse(accModCache[:]))
assertDatabaseAccount(accDelNoCache, nil)
assertDatabaseAccount(accDelCache, nil)
// assertDatabaseStorage ensures that a storage slot inside the database
// matches the given blob if it's already covered by the disk snapshot,
// and does not exist otherwise.
assertDatabaseStorage := func(account common.Hash, slot common.Hash, data []byte) {
t.Helper()
blob := rawdb.ReadStorageSnapshot(db, account, slot)
if bytes.Compare(append(account[:], slot[:]...), genMarker) > 0 && blob != nil {
t.Fatalf("test %d: post-marker (%x) storage database access (%x:%x) succeeded: %x", i, genMarker, account, slot, blob)
}
if bytes.Compare(append(account[:], slot[:]...), genMarker) <= 0 && !bytes.Equal(blob, data) {
t.Fatalf("test %d: pre-marker (%x) storage database access (%x:%x) mismatch: have %x, want %x", i, genMarker, account, slot, blob, data)
}
}
assertDatabaseStorage(conNoModNoCache, conNoModNoCacheSlot, conNoModNoCacheSlot[:])
assertDatabaseStorage(conNoModCache, conNoModCacheSlot, conNoModCacheSlot[:])
assertDatabaseStorage(conModNoCache, conModNoCacheSlot, reverse(conModNoCacheSlot[:]))
assertDatabaseStorage(conModCache, conModCacheSlot, reverse(conModCacheSlot[:]))
assertDatabaseStorage(conDelNoCache, conDelNoCacheSlot, nil)
assertDatabaseStorage(conDelCache, conDelCacheSlot, nil)
assertDatabaseStorage(conNukeNoCache, conNukeNoCacheSlot, nil)
assertDatabaseStorage(conNukeCache, conNukeCacheSlot, nil)
}
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values, discarding anything
// after the in-progress generation marker.
//
// This test case is a tiny specialized case of TestDiskPartialMerge, which tests
// some very specific cornercases that random tests won't ever trigger.
func TestDiskMidAccountPartialMerge(t *testing.T) {
// TODO(@karalabe) ?
}
// TestDiskSeek tests that seek-operations work on the disk layer
func TestDiskSeek(t *testing.T) {
// Create some accounts in the disk layer
var db ethdb.Database
if dir, err := ioutil.TempDir("", "disklayer-test"); err != nil {
t.Fatal(err)
} else {
defer os.RemoveAll(dir)
diskdb, err := leveldb.New(dir, 256, 0, "")
if err != nil {
t.Fatal(err)
}
db = rawdb.NewDatabase(diskdb)
}
// Fill even keys [0,2,4...]
for i := 0; i < 0xff; i += 2 {
acc := common.Hash{byte(i)}
rawdb.WriteAccountSnapshot(db, acc, acc[:])
}
// Add an 'higher' key, with incorrect (higher) prefix
highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1}
db.Put(highKey, []byte{0xff, 0xff})
baseRoot := randomHash()
rawdb.WriteSnapshotRoot(db, baseRoot)
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
},
},
}
// Test some different seek positions
type testcase struct {
pos byte
expkey byte
}
var cases = []testcase{
{0xff, 0x55}, // this should exit immediately without checking key
{0x01, 0x02},
{0xfe, 0xfe},
{0xfd, 0xfe},
{0x00, 0x00},
}
for i, tc := range cases {
it, err := snaps.AccountIterator(baseRoot, common.Hash{tc.pos})
if err != nil {
t.Fatalf("case %d, error: %v", i, err)
}
count := 0
for it.Next() {
k, v, err := it.Hash()[0], it.Account()[0], it.Error()
if err != nil {
t.Fatalf("test %d, item %d, error: %v", i, count, err)
}
// First item in iterator should have the expected key
if count == 0 && k != tc.expkey {
t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey)
}
count++
if v != k {
t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k)
}
}
}
}

262
core/state/snapshot/generate.go Normal file
View file

@ -0,0 +1,262 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"encoding/binary"
"math/big"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
var (
// emptyRoot is the known root hash of an empty trie.
emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
// emptyCode is the known hash of the empty EVM bytecode.
emptyCode = crypto.Keccak256Hash(nil)
)
// generatorStats is a collection of statistics gathered by the snapshot generator
// for logging purposes.
type generatorStats struct {
wiping chan struct{} // Notification channel if wiping is in progress
origin uint64 // Origin prefix where generation started
start time.Time // Timestamp when generation started
accounts uint64 // Number of accounts indexed
slots uint64 // Number of storage slots indexed
storage common.StorageSize // Account and storage slot size
}
// Log creates an contextual log with the given message and the context pulled
// from the internally maintained statistics.
func (gs *generatorStats) Log(msg string, marker []byte) {
var ctx []interface{}
// Figure out whether we're after or within an account
switch len(marker) {
case common.HashLength:
ctx = append(ctx, []interface{}{"at", common.BytesToHash(marker)}...)
case 2 * common.HashLength:
ctx = append(ctx, []interface{}{
"in", common.BytesToHash(marker[:common.HashLength]),
"at", common.BytesToHash(marker[common.HashLength:]),
}...)
}
// Add the usual measurements
ctx = append(ctx, []interface{}{
"accounts", gs.accounts,
"slots", gs.slots,
"storage", gs.storage,
"elapsed", common.PrettyDuration(time.Since(gs.start)),
}...)
// Calculate the estimated indexing time based on current stats
if len(marker) > 0 {
if done := binary.BigEndian.Uint64(marker[:8]) - gs.origin; done > 0 {
left := math.MaxUint64 - binary.BigEndian.Uint64(marker[:8])
speed := done/uint64(time.Since(gs.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
ctx = append(ctx, []interface{}{
"eta", common.PrettyDuration(time.Duration(left/speed) * time.Millisecond),
}...)
}
}
log.Info(msg, ctx...)
}
// generateSnapshot regenerates a brand new snapshot based on an existing state
// database and head block asynchronously. The snapshot is returned immediately
// and generation is continued in the background until done.
func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, wiper chan struct{}) *diskLayer {
// Wipe any previously existing snapshot from the database if no wiper is
// currently in progress.
if wiper == nil {
wiper = wipeSnapshot(diskdb, true)
}
// Create a new disk layer with an initialized state marker at zero
rawdb.WriteSnapshotRoot(diskdb, root)
base := &diskLayer{
diskdb: diskdb,
triedb: triedb,
root: root,
cache: fastcache.New(cache * 1024 * 1024),
genMarker: []byte{}, // Initialized but empty!
genPending: make(chan struct{}),
genAbort: make(chan chan *generatorStats),
}
go base.generate(&generatorStats{wiping: wiper, start: time.Now()})
return base
}
// generate is a background thread that iterates over the state and storage tries,
// constructing the state snapshot. All the arguments are purely for statistics
// gethering and logging, since the method surfs the blocks as they arrive, often
// being restarted.
func (dl *diskLayer) generate(stats *generatorStats) {
// If a database wipe is in operation, wait until it's done
if stats.wiping != nil {
stats.Log("Wiper running, state snapshotting paused", dl.genMarker)
select {
// If wiper is done, resume normal mode of operation
case <-stats.wiping:
stats.wiping = nil
stats.start = time.Now()
// If generator was aboted during wipe, return
case abort := <-dl.genAbort:
abort <- stats
return
}
}
// Create an account and state iterator pointing to the current generator marker
accTrie, err := trie.NewSecure(dl.root, dl.triedb)
if err != nil {
// The account trie is missing (GC), surf the chain until one becomes available
stats.Log("Trie missing, state snapshotting paused", dl.genMarker)
abort := <-dl.genAbort
abort <- stats
return
}
stats.Log("Resuming state snapshot generation", dl.genMarker)
var accMarker []byte
if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
accMarker = dl.genMarker[:common.HashLength]
}
accIt := trie.NewIterator(accTrie.NodeIterator(accMarker))
batch := dl.diskdb.NewBatch()
// Iterate from the previous marker and continue generating the state snapshot
logged := time.Now()
for accIt.Next() {
// Retrieve the current account and flatten it into the internal format
accountHash := common.BytesToHash(accIt.Key)
var acc struct {
Nonce uint64
Balance *big.Int
Root common.Hash
CodeHash []byte
}
if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil {
log.Crit("Invalid account encountered during snapshot creation", "err", err)
}
data := AccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash)
// If the account is not yet in-progress, write it out
if accMarker == nil || !bytes.Equal(accountHash[:], accMarker) {
rawdb.WriteAccountSnapshot(batch, accountHash, data)
stats.storage += common.StorageSize(1 + common.HashLength + len(data))
stats.accounts++
}
// If we've exceeded our batch allowance or termination was requested, flush to disk
var abort chan *generatorStats
select {
case abort = <-dl.genAbort:
default:
}
if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
// Only write and set the marker if we actually did something useful
if batch.ValueSize() > 0 {
batch.Write()
batch.Reset()
dl.lock.Lock()
dl.genMarker = accountHash[:]
dl.lock.Unlock()
}
if abort != nil {
stats.Log("Aborting state snapshot generation", accountHash[:])
abort <- stats
return
}
}
// If the account is in-progress, continue where we left off (otherwise iterate all)
if acc.Root != emptyRoot {
storeTrie, err := trie.NewSecure(acc.Root, dl.triedb)
if err != nil {
log.Crit("Storage trie inaccessible for snapshot generation", "err", err)
}
var storeMarker []byte
if accMarker != nil && bytes.Equal(accountHash[:], accMarker) && len(dl.genMarker) > common.HashLength {
storeMarker = dl.genMarker[common.HashLength:]
}
storeIt := trie.NewIterator(storeTrie.NodeIterator(storeMarker))
for storeIt.Next() {
rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(storeIt.Key), storeIt.Value)
stats.storage += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value))
stats.slots++
// If we've exceeded our batch allowance or termination was requested, flush to disk
var abort chan *generatorStats
select {
case abort = <-dl.genAbort:
default:
}
if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
// Only write and set the marker if we actually did something useful
if batch.ValueSize() > 0 {
batch.Write()
batch.Reset()
dl.lock.Lock()
dl.genMarker = append(accountHash[:], storeIt.Key...)
dl.lock.Unlock()
}
if abort != nil {
stats.Log("Aborting state snapshot generation", append(accountHash[:], storeIt.Key...))
abort <- stats
return
}
}
}
}
if time.Since(logged) > 8*time.Second {
stats.Log("Generating state snapshot", accIt.Key)
logged = time.Now()
}
// Some account processed, unmark the marker
accMarker = nil
}
// Snapshot fully generated, set the marker to nil
if batch.ValueSize() > 0 {
batch.Write()
}
log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots,
"storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start)))
dl.lock.Lock()
dl.genMarker = nil
close(dl.genPending)
dl.lock.Unlock()
// Someone will be looking for us, wait it out
abort := <-dl.genAbort
abort <- nil
}

204
core/state/snapshot/iterator.go Normal file
View file

@ -0,0 +1,204 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"fmt"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
)
// AccountIterator is an iterator to step over all the accounts in a snapshot,
// which may or may npt be composed of multiple layers.
type AccountIterator interface {
// Next steps the iterator forward one element, returning false if exhausted,
// or an error if iteration failed for some reason (e.g. root being iterated
// becomes stale and garbage collected).
Next() bool
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
Error() error
// Hash returns the hash of the account the iterator is currently at.
Hash() common.Hash
// Account returns the RLP encoded slim account the iterator is currently at.
// An error will be returned if the iterator becomes invalid (e.g. snaph
Account() []byte
// Release releases associated resources. Release should always succeed and
// can be called multiple times without causing error.
Release()
}
// diffAccountIterator is an account iterator that steps over the accounts (both
// live and deleted) contained within a single diff layer. Higher order iterators
// will use the deleted accounts to skip deeper iterators.
type diffAccountIterator struct {
// curHash is the current hash the iterator is positioned on. The field is
// explicitly tracked since the referenced diff layer might go stale after
// the iterator was positioned and we don't want to fail accessing the old
// hash as long as the iterator is not touched any more.
curHash common.Hash
layer *diffLayer // Live layer to retrieve values from
keys []common.Hash // Keys left in the layer to iterate
fail error // Any failures encountered (stale)
}
// AccountIterator creates an account iterator over a single diff layer.
func (dl *diffLayer) AccountIterator(seek common.Hash) AccountIterator {
// Seek out the requested starting account
hashes := dl.AccountList()
index := sort.Search(len(hashes), func(i int) bool {
return bytes.Compare(seek[:], hashes[i][:]) < 0
})
// Assemble and returned the already seeked iterator
return &diffAccountIterator{
layer: dl,
keys: hashes[index:],
}
}
// Next steps the iterator forward one element, returning false if exhausted.
func (it *diffAccountIterator) Next() bool {
// If the iterator was already stale, consider it a programmer error. Although
// we could just return false here, triggering this path would probably mean
// somebody forgot to check for Error, so lets blow up instead of undefined
// behavior that's hard to debug.
if it.fail != nil {
panic(fmt.Sprintf("called Next of failed iterator: %v", it.fail))
}
// Stop iterating if all keys were exhausted
if len(it.keys) == 0 {
return false
}
if it.layer.Stale() {
it.fail, it.keys = ErrSnapshotStale, nil
return false
}
// Iterator seems to be still alive, retrieve and cache the live hash
it.curHash = it.keys[0]
// key cached, shift the iterator and notify the user of success
it.keys = it.keys[1:]
return true
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
func (it *diffAccountIterator) Error() error {
return it.fail
}
// Hash returns the hash of the account the iterator is currently at.
func (it *diffAccountIterator) Hash() common.Hash {
return it.curHash
}
// Account returns the RLP encoded slim account the iterator is currently at.
// This method may _fail_, if the underlying layer has been flattened between
// the call to Next and Acccount. That type of error will set it.Err.
// This method assumes that flattening does not delete elements from
// the accountdata mapping (writing nil into it is fine though), and will panic
// if elements have been deleted.
func (it *diffAccountIterator) Account() []byte {
it.layer.lock.RLock()
blob, ok := it.layer.accountData[it.curHash]
if !ok {
if _, ok := it.layer.destructSet[it.curHash]; ok {
return nil
}
panic(fmt.Sprintf("iterator referenced non-existent account: %x", it.curHash))
}
it.layer.lock.RUnlock()
if it.layer.Stale() {
it.fail, it.keys = ErrSnapshotStale, nil
}
return blob
}
// Release is a noop for diff account iterators as there are no held resources.
func (it *diffAccountIterator) Release() {}
// diskAccountIterator is an account iterator that steps over the live accounts
// contained within a disk layer.
type diskAccountIterator struct {
layer *diskLayer
it ethdb.Iterator
}
// AccountIterator creates an account iterator over a disk layer.
func (dl *diskLayer) AccountIterator(seek common.Hash) AccountIterator {
pos := common.TrimRightZeroes(seek[:])
return &diskAccountIterator{
layer: dl,
it: dl.diskdb.NewIterator(rawdb.SnapshotAccountPrefix, pos),
}
}
// Next steps the iterator forward one element, returning false if exhausted.
func (it *diskAccountIterator) Next() bool {
// If the iterator was already exhausted, don't bother
if it.it == nil {
return false
}
// Try to advance the iterator and release it if we reached the end
for {
if !it.it.Next() || !bytes.HasPrefix(it.it.Key(), rawdb.SnapshotAccountPrefix) {
it.it.Release()
it.it = nil
return false
}
if len(it.it.Key()) == len(rawdb.SnapshotAccountPrefix)+common.HashLength {
break
}
}
return true
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
//
// A diff layer is immutable after creation content wise and can always be fully
// iterated without error, so this method always returns nil.
func (it *diskAccountIterator) Error() error {
return it.it.Error()
}
// Hash returns the hash of the account the iterator is currently at.
func (it *diskAccountIterator) Hash() common.Hash {
return common.BytesToHash(it.it.Key())
}
// Account returns the RLP encoded slim account the iterator is currently at.
func (it *diskAccountIterator) Account() []byte {
return it.it.Value()
}
// Release releases the database snapshot held during iteration.
func (it *diskAccountIterator) Release() {
// The iterator is auto-released on exhaustion, so make sure it's still alive
if it.it != nil {
it.it.Release()
it.it = nil
}
}

115
core/state/snapshot/iterator_binary.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"github.com/ethereum/go-ethereum/common"
)
// binaryAccountIterator is a simplistic iterator to step over the accounts in
// a snapshot, which may or may npt be composed of multiple layers. Performance
// wise this iterator is slow, it's meant for cross validating the fast one,
type binaryAccountIterator struct {
a *diffAccountIterator
b AccountIterator
aDone bool
bDone bool
k common.Hash
fail error
}
// newBinaryAccountIterator creates a simplistic account iterator to step over
// all the accounts in a slow, but eaily verifiable way.
func (dl *diffLayer) newBinaryAccountIterator() AccountIterator {
parent, ok := dl.parent.(*diffLayer)
if !ok {
// parent is the disk layer
return dl.AccountIterator(common.Hash{})
}
l := &binaryAccountIterator{
a: dl.AccountIterator(common.Hash{}).(*diffAccountIterator),
b: parent.newBinaryAccountIterator(),
}
l.aDone = !l.a.Next()
l.bDone = !l.b.Next()
return l
}
// Next steps the iterator forward one element, returning false if exhausted,
// or an error if iteration failed for some reason (e.g. root being iterated
// becomes stale and garbage collected).
func (it *binaryAccountIterator) Next() bool {
if it.aDone && it.bDone {
return false
}
nextB := it.b.Hash()
first:
nextA := it.a.Hash()
if it.aDone {
it.bDone = !it.b.Next()
it.k = nextB
return true
}
if it.bDone {
it.aDone = !it.a.Next()
it.k = nextA
return true
}
if diff := bytes.Compare(nextA[:], nextB[:]); diff < 0 {
it.aDone = !it.a.Next()
it.k = nextA
return true
} else if diff == 0 {
// Now we need to advance one of them
it.aDone = !it.a.Next()
goto first
}
it.bDone = !it.b.Next()
it.k = nextB
return true
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
func (it *binaryAccountIterator) Error() error {
return it.fail
}
// Hash returns the hash of the account the iterator is currently at.
func (it *binaryAccountIterator) Hash() common.Hash {
return it.k
}
// Account returns the RLP encoded slim account the iterator is currently at, or
// nil if the iterated snapshot stack became stale (you can check Error after
// to see if it failed or not).
func (it *binaryAccountIterator) Account() []byte {
blob, err := it.a.layer.AccountRLP(it.k)
if err != nil {
it.fail = err
return nil
}
return blob
}
// Release recursively releases all the iterators in the stack.
func (it *binaryAccountIterator) Release() {
it.a.Release()
it.b.Release()
}

302
core/state/snapshot/iterator_fast.go Normal file
View file

@ -0,0 +1,302 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"fmt"
"sort"
"github.com/ethereum/go-ethereum/common"
)
// weightedAccountIterator is an account iterator with an assigned weight. It is
// used to prioritise which account is the correct one if multiple iterators find
// the same one (modified in multiple consecutive blocks).
type weightedAccountIterator struct {
it AccountIterator
priority int
}
// weightedAccountIterators is a set of iterators implementing the sort.Interface.
type weightedAccountIterators []*weightedAccountIterator
// Len implements sort.Interface, returning the number of active iterators.
func (its weightedAccountIterators) Len() int { return len(its) }
// Less implements sort.Interface, returning which of two iterators in the stack
// is before the other.
func (its weightedAccountIterators) Less(i, j int) bool {
// Order the iterators primarily by the account hashes
hashI := its[i].it.Hash()
hashJ := its[j].it.Hash()
switch bytes.Compare(hashI[:], hashJ[:]) {
case -1:
return true
case 1:
return false
}
// Same account in multiple layers, split by priority
return its[i].priority < its[j].priority
}
// Swap implements sort.Interface, swapping two entries in the iterator stack.
func (its weightedAccountIterators) Swap(i, j int) {
its[i], its[j] = its[j], its[i]
}
// fastAccountIterator is a more optimized multi-layer iterator which maintains a
// direct mapping of all iterators leading down to the bottom layer.
type fastAccountIterator struct {
tree *Tree // Snapshot tree to reinitialize stale sub-iterators with
root common.Hash // Root hash to reinitialize stale sub-iterators through
curAccount []byte
iterators weightedAccountIterators
initiated bool
fail error
}
// newFastAccountIterator creates a new hierarhical account iterator with one
// element per diff layer. The returned combo iterator can be used to walk over
// the entire snapshot diff stack simultaneously.
func newFastAccountIterator(tree *Tree, root common.Hash, seek common.Hash) (AccountIterator, error) {
snap := tree.Snapshot(root)
if snap == nil {
return nil, fmt.Errorf("unknown snapshot: %x", root)
}
fi := &fastAccountIterator{
tree: tree,
root: root,
}
current := snap.(snapshot)
for depth := 0; current != nil; depth++ {
fi.iterators = append(fi.iterators, &weightedAccountIterator{
it: current.AccountIterator(seek),
priority: depth,
})
current = current.Parent()
}
fi.init()
return fi, nil
}
// init walks over all the iterators and resolves any clashes between them, after
// which it prepares the stack for step-by-step iteration.
func (fi *fastAccountIterator) init() {
// Track which account hashes are iterators positioned on
var positioned = make(map[common.Hash]int)
// Position all iterators and track how many remain live
for i := 0; i < len(fi.iterators); i++ {
// Retrieve the first element and if it clashes with a previous iterator,
// advance either the current one or the old one. Repeat until nothing is
// clashing any more.
it := fi.iterators[i]
for {
// If the iterator is exhausted, drop it off the end
if !it.it.Next() {
it.it.Release()
last := len(fi.iterators) - 1
fi.iterators[i] = fi.iterators[last]
fi.iterators[last] = nil
fi.iterators = fi.iterators[:last]
i--
break
}
// The iterator is still alive, check for collisions with previous ones
hash := it.it.Hash()
if other, exist := positioned[hash]; !exist {
positioned[hash] = i
break
} else {
// Iterators collide, one needs to be progressed, use priority to
// determine which.
//
// This whole else-block can be avoided, if we instead
// do an initial priority-sort of the iterators. If we do that,
// then we'll only wind up here if a lower-priority (preferred) iterator
// has the same value, and then we will always just continue.
// However, it costs an extra sort, so it's probably not better
if fi.iterators[other].priority < it.priority {
// The 'it' should be progressed
continue
} else {
// The 'other' should be progressed, swap them
it = fi.iterators[other]
fi.iterators[other], fi.iterators[i] = fi.iterators[i], fi.iterators[other]
continue
}
}
}
}
// Re-sort the entire list
sort.Sort(fi.iterators)
fi.initiated = false
}
// Next steps the iterator forward one element, returning false if exhausted.
func (fi *fastAccountIterator) Next() bool {
if len(fi.iterators) == 0 {
return false
}
if !fi.initiated {
// Don't forward first time -- we had to 'Next' once in order to
// do the sorting already
fi.initiated = true
fi.curAccount = fi.iterators[0].it.Account()
if innerErr := fi.iterators[0].it.Error(); innerErr != nil {
fi.fail = innerErr
return false
}
if fi.curAccount != nil {
return true
}
// Implicit else: we've hit a nil-account, and need to fall through to the
// loop below to land on something non-nil
}
// If an account is deleted in one of the layers, the key will still be there,
// but the actual value will be nil. However, the iterator should not
// export nil-values (but instead simply omit the key), so we need to loop
// here until we either
// - get a non-nil value,
// - hit an error,
// - or exhaust the iterator
for {
if !fi.next(0) {
return false // exhausted
}
fi.curAccount = fi.iterators[0].it.Account()
if innerErr := fi.iterators[0].it.Error(); innerErr != nil {
fi.fail = innerErr
return false // error
}
if fi.curAccount != nil {
break // non-nil value found
}
}
return true
}
// next handles the next operation internally and should be invoked when we know
// that two elements in the list may have the same value.
//
// For example, if the iterated hashes become [2,3,5,5,8,9,10], then we should
// invoke next(3), which will call Next on elem 3 (the second '5') and will
// cascade along the list, applying the same operation if needed.
func (fi *fastAccountIterator) next(idx int) bool {
// If this particular iterator got exhausted, remove it and return true (the
// next one is surely not exhausted yet, otherwise it would have been removed
// already).
if it := fi.iterators[idx].it; !it.Next() {
it.Release()
fi.iterators = append(fi.iterators[:idx], fi.iterators[idx+1:]...)
return len(fi.iterators) > 0
}
// If there's noone left to cascade into, return
if idx == len(fi.iterators)-1 {
return true
}
// We next-ed the iterator at 'idx', now we may have to re-sort that element
var (
cur, next = fi.iterators[idx], fi.iterators[idx+1]
curHash, nextHash = cur.it.Hash(), next.it.Hash()
)
if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 {
// It is still in correct place
return true
} else if diff == 0 && cur.priority < next.priority {
// So still in correct place, but we need to iterate on the next
fi.next(idx + 1)
return true
}
// At this point, the iterator is in the wrong location, but the remaining
// list is sorted. Find out where to move the item.
clash := -1
index := sort.Search(len(fi.iterators), func(n int) bool {
// The iterator always advances forward, so anything before the old slot
// is known to be behind us, so just skip them altogether. This actually
// is an important clause since the sort order got invalidated.
if n < idx {
return false
}
if n == len(fi.iterators)-1 {
// Can always place an elem last
return true
}
nextHash := fi.iterators[n+1].it.Hash()
if diff := bytes.Compare(curHash[:], nextHash[:]); diff < 0 {
return true
} else if diff > 0 {
return false
}
// The elem we're placing it next to has the same value,
// so whichever winds up on n+1 will need further iteraton
clash = n + 1
return cur.priority < fi.iterators[n+1].priority
})
fi.move(idx, index)
if clash != -1 {
fi.next(clash)
}
return true
}
// move advances an iterator to another position in the list.
func (fi *fastAccountIterator) move(index, newpos int) {
elem := fi.iterators[index]
copy(fi.iterators[index:], fi.iterators[index+1:newpos+1])
fi.iterators[newpos] = elem
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit (e.g. snapshot stack becoming stale).
func (fi *fastAccountIterator) Error() error {
return fi.fail
}
// Hash returns the current key
func (fi *fastAccountIterator) Hash() common.Hash {
return fi.iterators[0].it.Hash()
}
// Account returns the current key
func (fi *fastAccountIterator) Account() []byte {
return fi.curAccount
}
// Release iterates over all the remaining live layer iterators and releases each
// of thme individually.
func (fi *fastAccountIterator) Release() {
for _, it := range fi.iterators {
it.it.Release()
}
fi.iterators = nil
}
// Debug is a convencience helper during testing
func (fi *fastAccountIterator) Debug() {
for _, it := range fi.iterators {
fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0])
}
fmt.Println()
}

662
core/state/snapshot/iterator_test.go Normal file
View file

@ -0,0 +1,662 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"encoding/binary"
"fmt"
"math/rand"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
)
// TestAccountIteratorBasics tests some simple single-layer iteration
func TestAccountIteratorBasics(t *testing.T) {
var (
destructs = make(map[common.Hash]struct{})
accounts = make(map[common.Hash][]byte)
storage = make(map[common.Hash]map[common.Hash][]byte)
)
// Fill up a parent
for i := 0; i < 100; i++ {
h := randomHash()
data := randomAccount()
accounts[h] = data
if rand.Intn(4) == 0 {
destructs[h] = struct{}{}
}
if rand.Intn(2) == 0 {
accStorage := make(map[common.Hash][]byte)
value := make([]byte, 32)
rand.Read(value)
accStorage[randomHash()] = value
storage[h] = accStorage
}
}
// Add some (identical) layers on top
parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
it := parent.AccountIterator(common.Hash{})
verifyIterator(t, 100, it)
}
type testIterator struct {
values []byte
}
func newTestIterator(values ...byte) *testIterator {
return &testIterator{values}
}
func (ti *testIterator) Seek(common.Hash) {
panic("implement me")
}
func (ti *testIterator) Next() bool {
ti.values = ti.values[1:]
return len(ti.values) > 0
}
func (ti *testIterator) Error() error {
return nil
}
func (ti *testIterator) Hash() common.Hash {
return common.BytesToHash([]byte{ti.values[0]})
}
func (ti *testIterator) Account() []byte {
return nil
}
func (ti *testIterator) Release() {}
func TestFastIteratorBasics(t *testing.T) {
type testCase struct {
lists [][]byte
expKeys []byte
}
for i, tc := range []testCase{
{lists: [][]byte{{0, 1, 8}, {1, 2, 8}, {2, 9}, {4},
{7, 14, 15}, {9, 13, 15, 16}},
expKeys: []byte{0, 1, 2, 4, 7, 8, 9, 13, 14, 15, 16}},
{lists: [][]byte{{0, 8}, {1, 2, 8}, {7, 14, 15}, {8, 9},
{9, 10}, {10, 13, 15, 16}},
expKeys: []byte{0, 1, 2, 7, 8, 9, 10, 13, 14, 15, 16}},
} {
var iterators []*weightedAccountIterator
for i, data := range tc.lists {
it := newTestIterator(data...)
iterators = append(iterators, &weightedAccountIterator{it, i})
}
fi := &fastAccountIterator{
iterators: iterators,
initiated: false,
}
count := 0
for fi.Next() {
if got, exp := fi.Hash()[31], tc.expKeys[count]; exp != got {
t.Errorf("tc %d, [%d]: got %d exp %d", i, count, got, exp)
}
count++
}
}
}
func verifyIterator(t *testing.T, expCount int, it AccountIterator) {
t.Helper()
var (
count = 0
last = common.Hash{}
)
for it.Next() {
hash := it.Hash()
if bytes.Compare(last[:], hash[:]) >= 0 {
t.Errorf("wrong order: %x >= %x", last, hash)
}
if it.Account() == nil {
t.Errorf("iterator returned nil-value for hash %x", hash)
}
count++
}
if count != expCount {
t.Errorf("iterator count mismatch: have %d, want %d", count, expCount)
}
if err := it.Error(); err != nil {
t.Errorf("iterator failed: %v", err)
}
}
// TestAccountIteratorTraversal tests some simple multi-layer iteration.
func TestAccountIteratorTraversal(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Stack three diff layers on top with various overlaps
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
randomAccountSet("0xcc", "0xf0", "0xff"), nil)
// Verify the single and multi-layer iterators
head := snaps.Snapshot(common.HexToHash("0x04"))
verifyIterator(t, 3, head.(snapshot).AccountIterator(common.Hash{}))
verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator())
it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
defer it.Release()
verifyIterator(t, 7, it)
}
// TestAccountIteratorTraversalValues tests some multi-layer iteration, where we
// also expect the correct values to show up.
func TestAccountIteratorTraversalValues(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Create a batch of account sets to seed subsequent layers with
var (
a = make(map[common.Hash][]byte)
b = make(map[common.Hash][]byte)
c = make(map[common.Hash][]byte)
d = make(map[common.Hash][]byte)
e = make(map[common.Hash][]byte)
f = make(map[common.Hash][]byte)
g = make(map[common.Hash][]byte)
h = make(map[common.Hash][]byte)
)
for i := byte(2); i < 0xff; i++ {
a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
if i > 20 && i%2 == 0 {
b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
}
if i%4 == 0 {
c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
}
if i%7 == 0 {
d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
}
if i%8 == 0 {
e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
}
if i > 50 || i < 85 {
f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
}
if i%64 == 0 {
g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
}
if i%128 == 0 {
h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
}
}
// Assemble a stack of snapshots from the account layers
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil)
snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil)
snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil)
snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil)
snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil)
snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil)
snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil)
snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil)
it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{})
defer it.Release()
head := snaps.Snapshot(common.HexToHash("0x09"))
for it.Next() {
hash := it.Hash()
want, err := head.AccountRLP(hash)
if err != nil {
t.Fatalf("failed to retrieve expected account: %v", err)
}
if have := it.Account(); !bytes.Equal(want, have) {
t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
}
}
}
// This testcase is notorious, all layers contain the exact same 200 accounts.
func TestAccountIteratorLargeTraversal(t *testing.T) {
// Create a custom account factory to recreate the same addresses
makeAccounts := func(num int) map[common.Hash][]byte {
accounts := make(map[common.Hash][]byte)
for i := 0; i < num; i++ {
h := common.Hash{}
binary.BigEndian.PutUint64(h[:], uint64(i+1))
accounts[h] = randomAccount()
}
return accounts
}
// Build up a large stack of snapshots
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
for i := 1; i < 128; i++ {
snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil)
}
// Iterate the entire stack and ensure everything is hit only once
head := snaps.Snapshot(common.HexToHash("0x80"))
verifyIterator(t, 200, head.(snapshot).AccountIterator(common.Hash{}))
verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator())
it, _ := snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{})
defer it.Release()
verifyIterator(t, 200, it)
}
// TestAccountIteratorFlattening tests what happens when we
// - have a live iterator on child C (parent C1 -> C2 .. CN)
// - flattens C2 all the way into CN
// - continues iterating
func TestAccountIteratorFlattening(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Create a stack of diffs on top
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
randomAccountSet("0xcc", "0xf0", "0xff"), nil)
// Create an iterator and flatten the data from underneath it
it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
defer it.Release()
if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil {
t.Fatalf("failed to flatten snapshot stack: %v", err)
}
//verifyIterator(t, 7, it)
}
func TestAccountIteratorSeek(t *testing.T) {
// Create a snapshot stack with some initial data
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
randomAccountSet("0xcc", "0xf0", "0xff"), nil)
// Construct various iterators and ensure their tranversal is correct
it, _ := snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xdd"))
defer it.Release()
verifyIterator(t, 3, it) // expected: ee, f0, ff
it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"))
defer it.Release()
verifyIterator(t, 3, it) // expected: ee, f0, ff
it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff"))
defer it.Release()
verifyIterator(t, 0, it) // expected: nothing
it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xbb"))
defer it.Release()
verifyIterator(t, 5, it) // expected: cc, dd, ee, f0, ff
it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xef"))
defer it.Release()
verifyIterator(t, 2, it) // expected: f0, ff
it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xf0"))
defer it.Release()
verifyIterator(t, 1, it) // expected: ff
it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff"))
defer it.Release()
verifyIterator(t, 0, it) // expected: nothing
}
// TestIteratorDeletions tests that the iterator behaves correct when there are
// deleted accounts (where the Account() value is nil). The iterator
// should not output any accounts or nil-values for those cases.
func TestIteratorDeletions(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Stack three diff layers on top with various overlaps
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"),
nil, randomAccountSet("0x11", "0x22", "0x33"), nil)
deleted := common.HexToHash("0x22")
destructed := map[common.Hash]struct{}{
deleted: struct{}{},
}
snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"),
destructed, randomAccountSet("0x11", "0x33"), nil)
snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"),
nil, randomAccountSet("0x33", "0x44", "0x55"), nil)
// The output should be 11,33,44,55
it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
// Do a quick check
verifyIterator(t, 4, it)
it.Release()
// And a more detailed verification that we indeed do not see '0x22'
it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
defer it.Release()
for it.Next() {
hash := it.Hash()
if it.Account() == nil {
t.Errorf("iterator returned nil-value for hash %x", hash)
}
if hash == deleted {
t.Errorf("expected deleted elem %x to not be returned by iterator", deleted)
}
}
}
// BenchmarkAccountIteratorTraversal is a bit a bit notorious -- all layers contain the
// exact same 200 accounts. That means that we need to process 2000 items, but
// only spit out 200 values eventually.
//
// The value-fetching benchmark is easy on the binary iterator, since it never has to reach
// down at any depth for retrieving the values -- all are on the toppmost layer
//
// BenchmarkAccountIteratorTraversal/binary_iterator_keys-6 2239 483674 ns/op
// BenchmarkAccountIteratorTraversal/binary_iterator_values-6 2403 501810 ns/op
// BenchmarkAccountIteratorTraversal/fast_iterator_keys-6 1923 677966 ns/op
// BenchmarkAccountIteratorTraversal/fast_iterator_values-6 1741 649967 ns/op
func BenchmarkAccountIteratorTraversal(b *testing.B) {
// Create a custom account factory to recreate the same addresses
makeAccounts := func(num int) map[common.Hash][]byte {
accounts := make(map[common.Hash][]byte)
for i := 0; i < num; i++ {
h := common.Hash{}
binary.BigEndian.PutUint64(h[:], uint64(i+1))
accounts[h] = randomAccount()
}
return accounts
}
// Build up a large stack of snapshots
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
for i := 1; i <= 100; i++ {
snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil)
}
// We call this once before the benchmark, so the creation of
// sorted accountlists are not included in the results.
head := snaps.Snapshot(common.HexToHash("0x65"))
head.(*diffLayer).newBinaryAccountIterator()
b.Run("binary iterator keys", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := 0
it := head.(*diffLayer).newBinaryAccountIterator()
for it.Next() {
got++
}
if exp := 200; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("binary iterator values", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := 0
it := head.(*diffLayer).newBinaryAccountIterator()
for it.Next() {
got++
head.(*diffLayer).accountRLP(it.Hash(), 0)
}
if exp := 200; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("fast iterator keys", func(b *testing.B) {
for i := 0; i < b.N; i++ {
it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
defer it.Release()
got := 0
for it.Next() {
got++
}
if exp := 200; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("fast iterator values", func(b *testing.B) {
for i := 0; i < b.N; i++ {
it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
defer it.Release()
got := 0
for it.Next() {
got++
it.Account()
}
if exp := 200; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
}
// BenchmarkAccountIteratorLargeBaselayer is a pretty realistic benchmark, where
// the baselayer is a lot larger than the upper layer.
//
// This is heavy on the binary iterator, which in most cases will have to
// call recursively 100 times for the majority of the values
//
// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(keys)-6 514 1971999 ns/op
// BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(values)-6 61 18997492 ns/op
// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(keys)-6 10000 114385 ns/op
// BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(values)-6 4047 296823 ns/op
func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) {
// Create a custom account factory to recreate the same addresses
makeAccounts := func(num int) map[common.Hash][]byte {
accounts := make(map[common.Hash][]byte)
for i := 0; i < num; i++ {
h := common.Hash{}
binary.BigEndian.PutUint64(h[:], uint64(i+1))
accounts[h] = randomAccount()
}
return accounts
}
// Build up a large stack of snapshots
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil)
for i := 2; i <= 100; i++ {
snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil)
}
// We call this once before the benchmark, so the creation of
// sorted accountlists are not included in the results.
head := snaps.Snapshot(common.HexToHash("0x65"))
head.(*diffLayer).newBinaryAccountIterator()
b.Run("binary iterator (keys)", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := 0
it := head.(*diffLayer).newBinaryAccountIterator()
for it.Next() {
got++
}
if exp := 2000; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("binary iterator (values)", func(b *testing.B) {
for i := 0; i < b.N; i++ {
got := 0
it := head.(*diffLayer).newBinaryAccountIterator()
for it.Next() {
got++
v := it.Hash()
head.(*diffLayer).accountRLP(v, 0)
}
if exp := 2000; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("fast iterator (keys)", func(b *testing.B) {
for i := 0; i < b.N; i++ {
it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
defer it.Release()
got := 0
for it.Next() {
got++
}
if exp := 2000; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
b.Run("fast iterator (values)", func(b *testing.B) {
for i := 0; i < b.N; i++ {
it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
defer it.Release()
got := 0
for it.Next() {
it.Account()
got++
}
if exp := 2000; got != exp {
b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
}
}
})
}
/*
func BenchmarkBinaryAccountIteration(b *testing.B) {
benchmarkAccountIteration(b, func(snap snapshot) AccountIterator {
return snap.(*diffLayer).newBinaryAccountIterator()
})
}
func BenchmarkFastAccountIteration(b *testing.B) {
benchmarkAccountIteration(b, newFastAccountIterator)
}
func benchmarkAccountIteration(b *testing.B, iterator func(snap snapshot) AccountIterator) {
// Create a diff stack and randomize the accounts across them
layers := make([]map[common.Hash][]byte, 128)
for i := 0; i < len(layers); i++ {
layers[i] = make(map[common.Hash][]byte)
}
for i := 0; i < b.N; i++ {
depth := rand.Intn(len(layers))
layers[depth][randomHash()] = randomAccount()
}
stack := snapshot(emptyLayer())
for _, layer := range layers {
stack = stack.Update(common.Hash{}, layer, nil, nil)
}
// Reset the timers and report all the stats
it := iterator(stack)
b.ResetTimer()
b.ReportAllocs()
for it.Next() {
}
}
*/

262
core/state/snapshot/journal.go Normal file
View file

@ -0,0 +1,262 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// journalGenerator is a disk layer entry containing the generator progress marker.
type journalGenerator struct {
Wiping bool // Whether the database was in progress of being wiped
Done bool // Whether the generator finished creating the snapshot
Marker []byte
Accounts uint64
Slots uint64
Storage uint64
}
// journalDestruct is an account deletion entry in a diffLayer's disk journal.
type journalDestruct struct {
Hash common.Hash
}
// journalAccount is an account entry in a diffLayer's disk journal.
type journalAccount struct {
Hash common.Hash
Blob []byte
}
// journalStorage is an account's storage map in a diffLayer's disk journal.
type journalStorage struct {
Hash common.Hash
Keys []common.Hash
Vals [][]byte
}
// loadSnapshot loads a pre-existing state snapshot backed by a key-value store.
func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) (snapshot, error) {
// Retrieve the block number and hash of the snapshot, failing if no snapshot
// is present in the database (or crashed mid-update).
baseRoot := rawdb.ReadSnapshotRoot(diskdb)
if baseRoot == (common.Hash{}) {
return nil, errors.New("missing or corrupted snapshot")
}
base := &diskLayer{
diskdb: diskdb,
triedb: triedb,
cache: fastcache.New(cache * 1024 * 1024),
root: baseRoot,
}
// Retrieve the journal, it must exist since even for 0 layer it stores whether
// we've already generated the snapshot or are in progress only
journal := rawdb.ReadSnapshotJournal(diskdb)
if len(journal) == 0 {
return nil, errors.New("missing or corrupted snapshot journal")
}
r := rlp.NewStream(bytes.NewReader(journal), 0)
// Read the snapshot generation progress for the disk layer
var generator journalGenerator
if err := r.Decode(&generator); err != nil {
return nil, fmt.Errorf("failed to load snapshot progress marker: %v", err)
}
// Load all the snapshot diffs from the journal
snapshot, err := loadDiffLayer(base, r)
if err != nil {
return nil, err
}
// Entire snapshot journal loaded, sanity check the head and return
// Journal doesn't exist, don't worry if it's not supposed to
if head := snapshot.Root(); head != root {
return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root)
}
// Everything loaded correctly, resume any suspended operations
if !generator.Done {
// If the generator was still wiping, restart one from scratch (fine for
// now as it's rare and the wiper deletes the stuff it touches anyway, so
// restarting won't incur a lot of extra database hops.
var wiper chan struct{}
if generator.Wiping {
log.Info("Resuming previous snapshot wipe")
wiper = wipeSnapshot(diskdb, false)
}
// Whether or not wiping was in progress, load any generator progress too
base.genMarker = generator.Marker
if base.genMarker == nil {
base.genMarker = []byte{}
}
base.genPending = make(chan struct{})
base.genAbort = make(chan chan *generatorStats)
var origin uint64
if len(generator.Marker) >= 8 {
origin = binary.BigEndian.Uint64(generator.Marker)
}
go base.generate(&generatorStats{
wiping: wiper,
origin: origin,
start: time.Now(),
accounts: generator.Accounts,
slots: generator.Slots,
storage: common.StorageSize(generator.Storage),
})
}
return snapshot, nil
}
// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new
// diff and verifying that it can be linked to the requested parent.
func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
// Read the next diff journal entry
var root common.Hash
if err := r.Decode(&root); err != nil {
// The first read may fail with EOF, marking the end of the journal
if err == io.EOF {
return parent, nil
}
return nil, fmt.Errorf("load diff root: %v", err)
}
var destructs []journalDestruct
if err := r.Decode(&destructs); err != nil {
return nil, fmt.Errorf("load diff destructs: %v", err)
}
destructSet := make(map[common.Hash]struct{})
for _, entry := range destructs {
destructSet[entry.Hash] = struct{}{}
}
var accounts []journalAccount
if err := r.Decode(&accounts); err != nil {
return nil, fmt.Errorf("load diff accounts: %v", err)
}
accountData := make(map[common.Hash][]byte)
for _, entry := range accounts {
accountData[entry.Hash] = entry.Blob
}
var storage []journalStorage
if err := r.Decode(&storage); err != nil {
return nil, fmt.Errorf("load diff storage: %v", err)
}
storageData := make(map[common.Hash]map[common.Hash][]byte)
for _, entry := range storage {
slots := make(map[common.Hash][]byte)
for i, key := range entry.Keys {
slots[key] = entry.Vals[i]
}
storageData[entry.Hash] = slots
}
return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r)
}
// Journal writes the persistent layer generator stats into a buffer to be stored
// in the database as the snapshot journal.
func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
// If the snapshot is currently being generated, abort it
var stats *generatorStats
if dl.genAbort != nil {
abort := make(chan *generatorStats)
dl.genAbort <- abort
if stats = <-abort; stats != nil {
stats.Log("Journalling in-progress snapshot", dl.genMarker)
}
}
// Ensure the layer didn't get stale
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.stale {
return common.Hash{}, ErrSnapshotStale
}
// Write out the generator marker
entry := journalGenerator{
Done: dl.genMarker == nil,
Marker: dl.genMarker,
}
if stats != nil {
entry.Wiping = (stats.wiping != nil)
entry.Accounts = stats.accounts
entry.Slots = stats.slots
entry.Storage = uint64(stats.storage)
}
if err := rlp.Encode(buffer, entry); err != nil {
return common.Hash{}, err
}
return dl.root, nil
}
// Journal writes the memory layer contents into a buffer to be stored in the
// database as the snapshot journal.
func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
// Journal the parent first
base, err := dl.parent.Journal(buffer)
if err != nil {
return common.Hash{}, err
}
// Ensure the layer didn't get stale
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.Stale() {
return common.Hash{}, ErrSnapshotStale
}
// Everything below was journalled, persist this layer too
if err := rlp.Encode(buffer, dl.root); err != nil {
return common.Hash{}, err
}
destructs := make([]journalDestruct, 0, len(dl.destructSet))
for hash := range dl.destructSet {
destructs = append(destructs, journalDestruct{Hash: hash})
}
if err := rlp.Encode(buffer, destructs); err != nil {
return common.Hash{}, err
}
accounts := make([]journalAccount, 0, len(dl.accountData))
for hash, blob := range dl.accountData {
accounts = append(accounts, journalAccount{Hash: hash, Blob: blob})
}
if err := rlp.Encode(buffer, accounts); err != nil {
return common.Hash{}, err
}
storage := make([]journalStorage, 0, len(dl.storageData))
for hash, slots := range dl.storageData {
keys := make([]common.Hash, 0, len(slots))
vals := make([][]byte, 0, len(slots))
for key, val := range slots {
keys = append(keys, key)
vals = append(vals, val)
}
storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals})
}
if err := rlp.Encode(buffer, storage); err != nil {
return common.Hash{}, err
}
return base, nil
}

603
core/state/snapshot/snapshot.go Normal file
View file

@ -0,0 +1,603 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package snapshot implements a journalled, dynamic state dump.
package snapshot
import (
"bytes"
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/trie"
)
var (
snapshotCleanAccountHitMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/hit", nil)
snapshotCleanAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/miss", nil)
snapshotCleanAccountInexMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/inex", nil)
snapshotCleanAccountReadMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/read", nil)
snapshotCleanAccountWriteMeter = metrics.NewRegisteredMeter("state/snapshot/clean/account/write", nil)
snapshotCleanStorageHitMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/hit", nil)
snapshotCleanStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/miss", nil)
snapshotCleanStorageInexMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/inex", nil)
snapshotCleanStorageReadMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/read", nil)
snapshotCleanStorageWriteMeter = metrics.NewRegisteredMeter("state/snapshot/clean/storage/write", nil)
snapshotDirtyAccountHitMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/hit", nil)
snapshotDirtyAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/miss", nil)
snapshotDirtyAccountInexMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/inex", nil)
snapshotDirtyAccountReadMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/read", nil)
snapshotDirtyAccountWriteMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/account/write", nil)
snapshotDirtyStorageHitMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/hit", nil)
snapshotDirtyStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/miss", nil)
snapshotDirtyStorageInexMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/inex", nil)
snapshotDirtyStorageReadMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/read", nil)
snapshotDirtyStorageWriteMeter = metrics.NewRegisteredMeter("state/snapshot/dirty/storage/write", nil)
snapshotDirtyAccountHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/account/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015))
snapshotDirtyStorageHitDepthHist = metrics.NewRegisteredHistogram("state/snapshot/dirty/storage/hit/depth", nil, metrics.NewExpDecaySample(1028, 0.015))
snapshotFlushAccountItemMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/item", nil)
snapshotFlushAccountSizeMeter = metrics.NewRegisteredMeter("state/snapshot/flush/account/size", nil)
snapshotFlushStorageItemMeter = metrics.NewRegisteredMeter("state/snapshot/flush/storage/item", nil)
snapshotFlushStorageSizeMeter = metrics.NewRegisteredMeter("state/snapshot/flush/storage/size", nil)
snapshotBloomIndexTimer = metrics.NewRegisteredResettingTimer("state/snapshot/bloom/index", nil)
snapshotBloomErrorGauge = metrics.NewRegisteredGaugeFloat64("state/snapshot/bloom/error", nil)
snapshotBloomAccountTrueHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/truehit", nil)
snapshotBloomAccountFalseHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/falsehit", nil)
snapshotBloomAccountMissMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/account/miss", nil)
snapshotBloomStorageTrueHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/truehit", nil)
snapshotBloomStorageFalseHitMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/falsehit", nil)
snapshotBloomStorageMissMeter = metrics.NewRegisteredMeter("state/snapshot/bloom/storage/miss", nil)
// ErrSnapshotStale is returned from data accessors if the underlying snapshot
// layer had been invalidated due to the chain progressing forward far enough
// to not maintain the layer's original state.
ErrSnapshotStale = errors.New("snapshot stale")
// ErrNotCoveredYet is returned from data accessors if the underlying snapshot
// is being generated currently and the requested data item is not yet in the
// range of accounts covered.
ErrNotCoveredYet = errors.New("not covered yet")
// errSnapshotCycle is returned if a snapshot is attempted to be inserted
// that forms a cycle in the snapshot tree.
errSnapshotCycle = errors.New("snapshot cycle")
)
// Snapshot represents the functionality supported by a snapshot storage layer.
type Snapshot interface {
// Root returns the root hash for which this snapshot was made.
Root() common.Hash
// Account directly retrieves the account associated with a particular hash in
// the snapshot slim data format.
Account(hash common.Hash) (*Account, error)
// AccountRLP directly retrieves the account RLP associated with a particular
// hash in the snapshot slim data format.
AccountRLP(hash common.Hash) ([]byte, error)
// Storage directly retrieves the storage data associated with a particular hash,
// within a particular account.
Storage(accountHash, storageHash common.Hash) ([]byte, error)
}
// snapshot is the internal version of the snapshot data layer that supports some
// additional methods compared to the public API.
type snapshot interface {
Snapshot
// Parent returns the subsequent layer of a snapshot, or nil if the base was
// reached.
//
// Note, the method is an internal helper to avoid type switching between the
// disk and diff layers. There is no locking involved.
Parent() snapshot
// Update creates a new layer on top of the existing snapshot diff tree with
// the specified data items.
//
// Note, the maps are retained by the method to avoid copying everything.
Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer
// Journal commits an entire diff hierarchy to disk into a single journal entry.
// This is meant to be used during shutdown to persist the snapshot without
// flattening everything down (bad for reorgs).
Journal(buffer *bytes.Buffer) (common.Hash, error)
// Stale return whether this layer has become stale (was flattened across) or
// if it's still live.
Stale() bool
// AccountIterator creates an account iterator over an arbitrary layer.
AccountIterator(seek common.Hash) AccountIterator
}
// SnapshotTree is an Ethereum state snapshot tree. It consists of one persistent
// base layer backed by a key-value store, on top of which arbitrarily many in-
// memory diff layers are topped. The memory diffs can form a tree with branching,
// but the disk layer is singleton and common to all. If a reorg goes deeper than
// the disk layer, everything needs to be deleted.
//
// The goal of a state snapshot is twofold: to allow direct access to account and
// storage data to avoid expensive multi-level trie lookups; and to allow sorted,
// cheap iteration of the account/storage tries for sync aid.
type Tree struct {
diskdb ethdb.KeyValueStore // Persistent database to store the snapshot
triedb *trie.Database // In-memory cache to access the trie through
cache int // Megabytes permitted to use for read caches
layers map[common.Hash]snapshot // Collection of all known layers
lock sync.RWMutex
}
// New attempts to load an already existing snapshot from a persistent key-value
// store (with a number of memory layers from a journal), ensuring that the head
// of the snapshot matches the expected one.
//
// If the snapshot is missing or inconsistent, the entirety is deleted and will
// be reconstructed from scratch based on the tries in the key-value store, on a
// background thread.
func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool) *Tree {
// Create a new, empty snapshot tree
snap := &Tree{
diskdb: diskdb,
triedb: triedb,
cache: cache,
layers: make(map[common.Hash]snapshot),
}
if !async {
defer snap.waitBuild()
}
// Attempt to load a previously persisted snapshot and rebuild one if failed
head, err := loadSnapshot(diskdb, triedb, cache, root)
if err != nil {
log.Warn("Failed to load snapshot, regenerating", "err", err)
snap.Rebuild(root)
return snap
}
// Existing snapshot loaded, seed all the layers
for head != nil {
snap.layers[head.Root()] = head
head = head.Parent()
}
return snap
}
// waitBuild blocks until the snapshot finishes rebuilding. This method is meant
// to be used by tests to ensure we're testing what we believe we are.
func (t *Tree) waitBuild() {
// Find the rebuild termination channel
var done chan struct{}
t.lock.RLock()
for _, layer := range t.layers {
if layer, ok := layer.(*diskLayer); ok {
done = layer.genPending
break
}
}
t.lock.RUnlock()
// Wait until the snapshot is generated
if done != nil {
<-done
}
}
// Snapshot retrieves a snapshot belonging to the given block root, or nil if no
// snapshot is maintained for that block.
func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot {
t.lock.RLock()
defer t.lock.RUnlock()
return t.layers[blockRoot]
}
// Update adds a new snapshot into the tree, if that can be linked to an existing
// old parent. It is disallowed to insert a disk layer (the origin of all).
func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {
// Reject noop updates to avoid self-loops in the snapshot tree. This is a
// special case that can only happen for Clique networks where empty blocks
// don't modify the state (0 block subsidy).
//
// Although we could silently ignore this internally, it should be the caller's
// responsibility to avoid even attempting to insert such a snapshot.
if blockRoot == parentRoot {
return errSnapshotCycle
}
// Generate a new snapshot on top of the parent
parent := t.Snapshot(parentRoot).(snapshot)
if parent == nil {
return fmt.Errorf("parent [%#x] snapshot missing", parentRoot)
}
snap := parent.Update(blockRoot, destructs, accounts, storage)
// Save the new snapshot for later
t.lock.Lock()
defer t.lock.Unlock()
t.layers[snap.root] = snap
return nil
}
// Cap traverses downwards the snapshot tree from a head block hash until the
// number of allowed layers are crossed. All layers beyond the permitted number
// are flattened downwards.
func (t *Tree) Cap(root common.Hash, layers int) error {
// Retrieve the head snapshot to cap from
snap := t.Snapshot(root)
if snap == nil {
return fmt.Errorf("snapshot [%#x] missing", root)
}
diff, ok := snap.(*diffLayer)
if !ok {
return fmt.Errorf("snapshot [%#x] is disk layer", root)
}
// Run the internal capping and discard all stale layers
t.lock.Lock()
defer t.lock.Unlock()
// Flattening the bottom-most diff layer requires special casing since there's
// no child to rewire to the grandparent. In that case we can fake a temporary
// child for the capping and then remove it.
var persisted *diskLayer
switch layers {
case 0:
// If full commit was requested, flatten the diffs and merge onto disk
diff.lock.RLock()
base := diffToDisk(diff.flatten().(*diffLayer))
diff.lock.RUnlock()
// Replace the entire snapshot tree with the flat base
t.layers = map[common.Hash]snapshot{base.root: base}
return nil
case 1:
// If full flattening was requested, flatten the diffs but only merge if the
// memory limit was reached
var (
bottom *diffLayer
base *diskLayer
)
diff.lock.RLock()
bottom = diff.flatten().(*diffLayer)
if bottom.memory >= aggregatorMemoryLimit {
base = diffToDisk(bottom)
}
diff.lock.RUnlock()
// If all diff layers were removed, replace the entire snapshot tree
if base != nil {
t.layers = map[common.Hash]snapshot{base.root: base}
return nil
}
// Merge the new aggregated layer into the snapshot tree, clean stales below
t.layers[bottom.root] = bottom
default:
// Many layers requested to be retained, cap normally
persisted = t.cap(diff, layers)
}
// Remove any layer that is stale or links into a stale layer
children := make(map[common.Hash][]common.Hash)
for root, snap := range t.layers {
if diff, ok := snap.(*diffLayer); ok {
parent := diff.parent.Root()
children[parent] = append(children[parent], root)
}
}
var remove func(root common.Hash)
remove = func(root common.Hash) {
delete(t.layers, root)
for _, child := range children[root] {
remove(child)
}
delete(children, root)
}
for root, snap := range t.layers {
if snap.Stale() {
remove(root)
}
}
// If the disk layer was modified, regenerate all the cummulative blooms
if persisted != nil {
var rebloom func(root common.Hash)
rebloom = func(root common.Hash) {
if diff, ok := t.layers[root].(*diffLayer); ok {
diff.rebloom(persisted)
}
for _, child := range children[root] {
rebloom(child)
}
}
rebloom(persisted.root)
}
return nil
}
// cap traverses downwards the diff tree until the number of allowed layers are
// crossed. All diffs beyond the permitted number are flattened downwards. If the
// layer limit is reached, memory cap is also enforced (but not before).
//
// The method returns the new disk layer if diffs were persistend into it.
func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer {
// Dive until we run out of layers or reach the persistent database
for ; layers > 2; layers-- {
// If we still have diff layers below, continue down
if parent, ok := diff.parent.(*diffLayer); ok {
diff = parent
} else {
// Diff stack too shallow, return without modifications
return nil
}
}
// We're out of layers, flatten anything below, stopping if it's the disk or if
// the memory limit is not yet exceeded.
switch parent := diff.parent.(type) {
case *diskLayer:
return nil
case *diffLayer:
// Flatten the parent into the grandparent. The flattening internally obtains a
// write lock on grandparent.
flattened := parent.flatten().(*diffLayer)
t.layers[flattened.root] = flattened
diff.lock.Lock()
defer diff.lock.Unlock()
diff.parent = flattened
if flattened.memory < aggregatorMemoryLimit {
// Accumulator layer is smaller than the limit, so we can abort, unless
// there's a snapshot being generated currently. In that case, the trie
// will move fron underneath the generator so we **must** merge all the
// partial data down into the snapshot and restart the generation.
if flattened.parent.(*diskLayer).genAbort == nil {
return nil
}
}
default:
panic(fmt.Sprintf("unknown data layer: %T", parent))
}
// If the bottom-most layer is larger than our memory cap, persist to disk
bottom := diff.parent.(*diffLayer)
bottom.lock.RLock()
base := diffToDisk(bottom)
bottom.lock.RUnlock()
t.layers[base.root] = base
diff.parent = base
return base
}
// diffToDisk merges a bottom-most diff into the persistent disk layer underneath
// it. The method will panic if called onto a non-bottom-most diff layer.
func diffToDisk(bottom *diffLayer) *diskLayer {
var (
base = bottom.parent.(*diskLayer)
batch = base.diskdb.NewBatch()
stats *generatorStats
)
// If the disk layer is running a snapshot generator, abort it
if base.genAbort != nil {
abort := make(chan *generatorStats)
base.genAbort <- abort
stats = <-abort
}
// Start by temporarily deleting the current snapshot block marker. This
// ensures that in the case of a crash, the entire snapshot is invalidated.
rawdb.DeleteSnapshotRoot(batch)
// Mark the original base as stale as we're going to create a new wrapper
base.lock.Lock()
if base.stale {
panic("parent disk layer is stale") // we've committed into the same base from two children, boo
}
base.stale = true
base.lock.Unlock()
// Destroy all the destructed accounts from the database
for hash := range bottom.destructSet {
// Skip any account not covered yet by the snapshot
if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 {
continue
}
// Remove all storage slots
rawdb.DeleteAccountSnapshot(batch, hash)
base.cache.Set(hash[:], nil)
it := rawdb.IterateStorageSnapshots(base.diskdb, hash)
for it.Next() {
if key := it.Key(); len(key) == 65 { // TODO(karalabe): Yuck, we should move this into the iterator
batch.Delete(key)
base.cache.Del(key[1:])
snapshotFlushStorageItemMeter.Mark(1)
}
}
it.Release()
}
// Push all updated accounts into the database
for hash, data := range bottom.accountData {
// Skip any account not covered yet by the snapshot
if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 {
continue
}
// Push the account to disk
rawdb.WriteAccountSnapshot(batch, hash, data)
base.cache.Set(hash[:], data)
snapshotCleanAccountWriteMeter.Mark(int64(len(data)))
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write account snapshot", "err", err)
}
batch.Reset()
}
snapshotFlushAccountItemMeter.Mark(1)
snapshotFlushAccountSizeMeter.Mark(int64(len(data)))
}
// Push all the storage slots into the database
for accountHash, storage := range bottom.storageData {
// Skip any account not covered yet by the snapshot
if base.genMarker != nil && bytes.Compare(accountHash[:], base.genMarker) > 0 {
continue
}
// Generation might be mid-account, track that case too
midAccount := base.genMarker != nil && bytes.Equal(accountHash[:], base.genMarker[:common.HashLength])
for storageHash, data := range storage {
// Skip any slot not covered yet by the snapshot
if midAccount && bytes.Compare(storageHash[:], base.genMarker[common.HashLength:]) > 0 {
continue
}
if len(data) > 0 {
rawdb.WriteStorageSnapshot(batch, accountHash, storageHash, data)
base.cache.Set(append(accountHash[:], storageHash[:]...), data)
snapshotCleanStorageWriteMeter.Mark(int64(len(data)))
} else {
rawdb.DeleteStorageSnapshot(batch, accountHash, storageHash)
base.cache.Set(append(accountHash[:], storageHash[:]...), nil)
}
snapshotFlushStorageItemMeter.Mark(1)
snapshotFlushStorageSizeMeter.Mark(int64(len(data)))
}
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write storage snapshot", "err", err)
}
batch.Reset()
}
}
// Update the snapshot block marker and write any remainder data
rawdb.WriteSnapshotRoot(batch, bottom.root)
if err := batch.Write(); err != nil {
log.Crit("Failed to write leftover snapshot", "err", err)
}
res := &diskLayer{
root: bottom.root,
cache: base.cache,
diskdb: base.diskdb,
triedb: base.triedb,
genMarker: base.genMarker,
genPending: base.genPending,
}
// If snapshot generation hasn't finished yet, port over all the starts and
// continue where the previous round left off.
//
// Note, the `base.genAbort` comparison is not used normally, it's checked
// to allow the tests to play with the marker without triggering this path.
if base.genMarker != nil && base.genAbort != nil {
res.genMarker = base.genMarker
res.genAbort = make(chan chan *generatorStats)
go res.generate(stats)
}
return res
}
// Journal commits an entire diff hierarchy to disk into a single journal entry.
// This is meant to be used during shutdown to persist the snapshot without
// flattening everything down (bad for reorgs).
//
// The method returns the root hash of the base layer that needs to be persisted
// to disk as a trie too to allow continuing any pending generation op.
func (t *Tree) Journal(root common.Hash) (common.Hash, error) {
// Retrieve the head snapshot to journal from var snap snapshot
snap := t.Snapshot(root)
if snap == nil {
return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root)
}
// Run the journaling
t.lock.Lock()
defer t.lock.Unlock()
journal := new(bytes.Buffer)
base, err := snap.(snapshot).Journal(journal)
if err != nil {
return common.Hash{}, err
}
// Store the journal into the database and return
rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes())
return base, nil
}
// Rebuild wipes all available snapshot data from the persistent database and
// discard all caches and diff layers. Afterwards, it starts a new snapshot
// generator with the given root hash.
func (t *Tree) Rebuild(root common.Hash) {
t.lock.Lock()
defer t.lock.Unlock()
// Track whether there's a wipe currently running and keep it alive if so
var wiper chan struct{}
// Iterate over and mark all layers stale
for _, layer := range t.layers {
switch layer := layer.(type) {
case *diskLayer:
// If the base layer is generating, abort it and save
if layer.genAbort != nil {
abort := make(chan *generatorStats)
layer.genAbort <- abort
if stats := <-abort; stats != nil {
wiper = stats.wiping
}
}
// Layer should be inactive now, mark it as stale
layer.lock.Lock()
layer.stale = true
layer.lock.Unlock()
case *diffLayer:
// If the layer is a simple diff, simply mark as stale
layer.lock.Lock()
atomic.StoreUint32(&layer.stale, 1)
layer.lock.Unlock()
default:
panic(fmt.Sprintf("unknown layer type: %T", layer))
}
}
// Start generating a new snapshot from scratch on a backgroung thread. The
// generator will run a wiper first if there's not one running right now.
log.Info("Rebuilding state snapshot")
t.layers = map[common.Hash]snapshot{
root: generateSnapshot(t.diskdb, t.triedb, t.cache, root, wiper),
}
}
// AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash.
func (t *Tree) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {
return newFastAccountIterator(t, root, seek)
}

348
core/state/snapshot/snapshot_test.go Normal file
View file

@ -0,0 +1,348 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"fmt"
"math/big"
"math/rand"
"testing"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/rlp"
)
// randomHash generates a random blob of data and returns it as a hash.
func randomHash() common.Hash {
var hash common.Hash
if n, err := rand.Read(hash[:]); n != common.HashLength || err != nil {
panic(err)
}
return hash
}
// randomAccount generates a random account and returns it RLP encoded.
func randomAccount() []byte {
root := randomHash()
a := Account{
Balance: big.NewInt(rand.Int63()),
Nonce: rand.Uint64(),
Root: root[:],
CodeHash: emptyCode[:],
}
data, _ := rlp.EncodeToBytes(a)
return data
}
// randomAccountSet generates a set of random accounts with the given strings as
// the account address hashes.
func randomAccountSet(hashes ...string) map[common.Hash][]byte {
accounts := make(map[common.Hash][]byte)
for _, hash := range hashes {
accounts[common.HexToHash(hash)] = randomAccount()
}
return accounts
}
// Tests that if a disk layer becomes stale, no active external references will
// be returned with junk data. This version of the test flattens every diff layer
// to check internal corner case around the bottom-most memory accumulator.
func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Retrieve a reference to the base and commit a diff on top
ref := snaps.Snapshot(base.root)
accounts := map[common.Hash][]byte{
common.HexToHash("0xa1"): randomAccount(),
}
if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if n := len(snaps.layers); n != 2 {
t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 2)
}
// Commit the diff layer onto the disk and ensure it's persisted
if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil {
t.Fatalf("failed to merge diff layer onto disk: %v", err)
}
// Since the base layer was modified, ensure that data retrieval on the external reference fail
if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
}
if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
}
if n := len(snaps.layers); n != 1 {
t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 1)
fmt.Println(snaps.layers)
}
}
// Tests that if a disk layer becomes stale, no active external references will
// be returned with junk data. This version of the test retains the bottom diff
// layer to check the usual mode of operation where the accumulator is retained.
func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Retrieve a reference to the base and commit two diffs on top
ref := snaps.Snapshot(base.root)
accounts := map[common.Hash][]byte{
common.HexToHash("0xa1"): randomAccount(),
}
if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if n := len(snaps.layers); n != 3 {
t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3)
}
// Commit the diff layer onto the disk and ensure it's persisted
defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit)
aggregatorMemoryLimit = 0
if err := snaps.Cap(common.HexToHash("0x03"), 2); err != nil {
t.Fatalf("failed to merge diff layer onto disk: %v", err)
}
// Since the base layer was modified, ensure that data retrievald on the external reference fail
if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
}
if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
}
if n := len(snaps.layers); n != 2 {
t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2)
fmt.Println(snaps.layers)
}
}
// Tests that if a diff layer becomes stale, no active external references will
// be returned with junk data. This version of the test flattens every diff layer
// to check internal corner case around the bottom-most memory accumulator.
func TestDiffLayerExternalInvalidationFullFlatten(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Commit two diffs on top and retrieve a reference to the bottommost
accounts := map[common.Hash][]byte{
common.HexToHash("0xa1"): randomAccount(),
}
if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if n := len(snaps.layers); n != 3 {
t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3)
}
ref := snaps.Snapshot(common.HexToHash("0x02"))
// Flatten the diff layer into the bottom accumulator
if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil {
t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
}
// Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail
if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
}
if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
}
if n := len(snaps.layers); n != 2 {
t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2)
fmt.Println(snaps.layers)
}
}
// Tests that if a diff layer becomes stale, no active external references will
// be returned with junk data. This version of the test retains the bottom diff
// layer to check the usual mode of operation where the accumulator is retained.
func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) {
// Create an empty base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// Commit three diffs on top and retrieve a reference to the bottommost
accounts := map[common.Hash][]byte{
common.HexToHash("0xa1"): randomAccount(),
}
if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil {
t.Fatalf("failed to create a diff layer: %v", err)
}
if n := len(snaps.layers); n != 4 {
t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 4)
}
ref := snaps.Snapshot(common.HexToHash("0x02"))
// Doing a Cap operation with many allowed layers should be a no-op
exp := len(snaps.layers)
if err := snaps.Cap(common.HexToHash("0x04"), 2000); err != nil {
t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
}
if got := len(snaps.layers); got != exp {
t.Errorf("layers modified, got %d exp %d", got, exp)
}
// Flatten the diff layer into the bottom accumulator
if err := snaps.Cap(common.HexToHash("0x04"), 2); err != nil {
t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
}
// Since the accumulator diff layer was modified, ensure that data retrievald on the external reference fail
if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
}
if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
}
if n := len(snaps.layers); n != 3 {
t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 3)
fmt.Println(snaps.layers)
}
}
// TestPostCapBasicDataAccess tests some functionality regarding capping/flattening.
func TestPostCapBasicDataAccess(t *testing.T) {
// setAccount is a helper to construct a random account entry and assign it to
// an account slot in a snapshot
setAccount := func(accKey string) map[common.Hash][]byte {
return map[common.Hash][]byte{
common.HexToHash(accKey): randomAccount(),
}
}
// Create a starting base layer and a snapshot tree out of it
base := &diskLayer{
diskdb: rawdb.NewMemoryDatabase(),
root: common.HexToHash("0x01"),
cache: fastcache.New(1024 * 500),
}
snaps := &Tree{
layers: map[common.Hash]snapshot{
base.root: base,
},
}
// The lowest difflayer
snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil)
snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil)
snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil)
snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil)
snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil)
// checkExist verifies if an account exiss in a snapshot
checkExist := func(layer *diffLayer, key string) error {
if data, _ := layer.Account(common.HexToHash(key)); data == nil {
return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key))
}
return nil
}
// shouldErr checks that an account access errors as expected
shouldErr := func(layer *diffLayer, key string) error {
if data, err := layer.Account(common.HexToHash(key)); err == nil {
return fmt.Errorf("expected error, got data %x", data)
}
return nil
}
// check basics
snap := snaps.Snapshot(common.HexToHash("0xb3")).(*diffLayer)
if err := checkExist(snap, "0xa1"); err != nil {
t.Error(err)
}
if err := checkExist(snap, "0xb2"); err != nil {
t.Error(err)
}
if err := checkExist(snap, "0xb3"); err != nil {
t.Error(err)
}
// Cap to a bad root should fail
if err := snaps.Cap(common.HexToHash("0x1337"), 0); err == nil {
t.Errorf("expected error, got none")
}
// Now, merge the a-chain
snaps.Cap(common.HexToHash("0xa3"), 0)
// At this point, a2 got merged into a1. Thus, a1 is now modified, and as a1 is
// the parent of b2, b2 should no longer be able to iterate into parent.
// These should still be accessible
if err := checkExist(snap, "0xb2"); err != nil {
t.Error(err)
}
if err := checkExist(snap, "0xb3"); err != nil {
t.Error(err)
}
// But these would need iteration into the modified parent
if err := shouldErr(snap, "0xa1"); err != nil {
t.Error(err)
}
if err := shouldErr(snap, "0xa2"); err != nil {
t.Error(err)
}
if err := shouldErr(snap, "0xa3"); err != nil {
t.Error(err)
}
// Now, merge it again, just for fun. It should now error, since a3
// is a disk layer
if err := snaps.Cap(common.HexToHash("0xa3"), 0); err == nil {
t.Error("expected error capping the disk layer, got none")
}
}

36
core/state/snapshot/sort.go Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"github.com/ethereum/go-ethereum/common"
)
// hashes is a helper to implement sort.Interface.
type hashes []common.Hash
// Len is the number of elements in the collection.
func (hs hashes) Len() int { return len(hs) }
// Less reports whether the element with index i should sort before the element
// with index j.
func (hs hashes) Less(i, j int) bool { return bytes.Compare(hs[i][:], hs[j][:]) < 0 }
// Swap swaps the elements with indexes i and j.
func (hs hashes) Swap(i, j int) { hs[i], hs[j] = hs[j], hs[i] }

131
core/state/snapshot/wipe.go Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"bytes"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
// wipeSnapshot starts a goroutine to iterate over the entire key-value database
// and delete all the data associated with the snapshot (accounts, storage,
// metadata). After all is done, the snapshot range of the database is compacted
// to free up unused data blocks.
func wipeSnapshot(db ethdb.KeyValueStore, full bool) chan struct{} {
// Wipe the snapshot root marker synchronously
if full {
rawdb.DeleteSnapshotRoot(db)
}
// Wipe everything else asynchronously
wiper := make(chan struct{}, 1)
go func() {
if err := wipeContent(db); err != nil {
log.Error("Failed to wipe state snapshot", "err", err) // Database close will trigger this
return
}
close(wiper)
}()
return wiper
}
// wipeContent iterates over the entire key-value database and deletes all the
// data associated with the snapshot (accounts, storage), but not the root hash
// as the wiper is meant to run on a background thread but the root needs to be
// removed in sync to avoid data races. After all is done, the snapshot range of
// the database is compacted to free up unused data blocks.
func wipeContent(db ethdb.KeyValueStore) error {
if err := wipeKeyRange(db, "accounts", rawdb.SnapshotAccountPrefix, len(rawdb.SnapshotAccountPrefix)+common.HashLength); err != nil {
return err
}
if err := wipeKeyRange(db, "storage", rawdb.SnapshotStoragePrefix, len(rawdb.SnapshotStoragePrefix)+2*common.HashLength); err != nil {
return err
}
// Compact the snapshot section of the database to get rid of unused space
start := time.Now()
log.Info("Compacting snapshot account area ")
end := common.CopyBytes(rawdb.SnapshotAccountPrefix)
end[len(end)-1]++
if err := db.Compact(rawdb.SnapshotAccountPrefix, end); err != nil {
return err
}
log.Info("Compacting snapshot storage area ")
end = common.CopyBytes(rawdb.SnapshotStoragePrefix)
end[len(end)-1]++
if err := db.Compact(rawdb.SnapshotStoragePrefix, end); err != nil {
return err
}
log.Info("Compacted snapshot area in database", "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}
// wipeKeyRange deletes a range of keys from the database starting with prefix
// and having a specific total key length.
func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int) error {
// Batch deletions together to avoid holding an iterator for too long
var (
batch = db.NewBatch()
items int
)
// Iterate over the key-range and delete all of them
start, logged := time.Now(), time.Now()
it := db.NewIterator(prefix, nil)
for it.Next() {
// Skip any keys with the correct prefix but wrong lenth (trie nodes)
key := it.Key()
if !bytes.HasPrefix(key, prefix) {
break
}
if len(key) != keylen {
continue
}
// Delete the key and periodically recreate the batch and iterator
batch.Delete(key)
items++
if items%10000 == 0 {
// Batch too large (or iterator too long lived, flush and recreate)
it.Release()
if err := batch.Write(); err != nil {
return err
}
batch.Reset()
seekPos := key[len(prefix):]
it = db.NewIterator(prefix, seekPos)
if time.Since(logged) > 8*time.Second {
log.Info("Deleting state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
}
}
it.Release()
if err := batch.Write(); err != nil {
return err
}
log.Info("Deleted state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}

124
core/state/snapshot/wipe_test.go Normal file
View file

@ -0,0 +1,124 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package snapshot
import (
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
// Tests that given a database with random data content, all parts of a snapshot
// can be crrectly wiped without touching anything else.
func TestWipe(t *testing.T) {
// Create a database with some random snapshot data
db := memorydb.New()
for i := 0; i < 128; i++ {
account := randomHash()
rawdb.WriteAccountSnapshot(db, account, randomHash().Bytes())
for j := 0; j < 1024; j++ {
rawdb.WriteStorageSnapshot(db, account, randomHash(), randomHash().Bytes())
}
}
rawdb.WriteSnapshotRoot(db, randomHash())
// Add some random non-snapshot data too to make wiping harder
for i := 0; i < 65536; i++ {
// Generate a key that's the wrong length for a state snapshot item
var keysize int
for keysize == 0 || keysize == 32 || keysize == 64 {
keysize = 8 + rand.Intn(64) // +8 to ensure we will "never" randomize duplicates
}
// Randomize the suffix, dedup and inject it under the snapshot namespace
keysuffix := make([]byte, keysize)
rand.Read(keysuffix)
if rand.Int31n(2) == 0 {
db.Put(append(rawdb.SnapshotAccountPrefix, keysuffix...), randomHash().Bytes())
} else {
db.Put(append(rawdb.SnapshotStoragePrefix, keysuffix...), randomHash().Bytes())
}
}
// Sanity check that all the keys are present
var items int
it := db.NewIterator(rawdb.SnapshotAccountPrefix, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if len(key) == len(rawdb.SnapshotAccountPrefix)+common.HashLength {
items++
}
}
it = db.NewIterator(rawdb.SnapshotStoragePrefix, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if len(key) == len(rawdb.SnapshotStoragePrefix)+2*common.HashLength {
items++
}
}
if items != 128+128*1024 {
t.Fatalf("snapshot size mismatch: have %d, want %d", items, 128+128*1024)
}
if hash := rawdb.ReadSnapshotRoot(db); hash == (common.Hash{}) {
t.Errorf("snapshot block marker mismatch: have %#x, want <not-nil>", hash)
}
// Wipe all snapshot entries from the database
<-wipeSnapshot(db, true)
// Iterate over the database end ensure no snapshot information remains
it = db.NewIterator(rawdb.SnapshotAccountPrefix, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if len(key) == len(rawdb.SnapshotAccountPrefix)+common.HashLength {
t.Errorf("snapshot entry remained after wipe: %x", key)
}
}
it = db.NewIterator(rawdb.SnapshotStoragePrefix, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if len(key) == len(rawdb.SnapshotStoragePrefix)+2*common.HashLength {
t.Errorf("snapshot entry remained after wipe: %x", key)
}
}
if hash := rawdb.ReadSnapshotRoot(db); hash != (common.Hash{}) {
t.Errorf("snapshot block marker remained after wipe: %#x", hash)
}
// Iterate over the database and ensure miscellaneous items are present
items = 0
it = db.NewIterator(nil, nil)
defer it.Release()
for it.Next() {
items++
}
if items != 65536 {
t.Fatalf("misc item count mismatch: have %d, want %d", items, 65536)
}
}

59
core/state/state_object.go
View file

@ -195,15 +195,35 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has
if value, cached := s.originStorage[key]; cached {
return value
}
// Track the amount of time wasted on reading the storage trie
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now())
// If no live objects are available, attempt to use snapshots
var (
enc []byte
err error
)
if s.db.snap != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.SnapshotStorageReads += time.Since(start) }(time.Now())
}
// If the object was destructed in *this* block (and potentially resurrected),
// the storage has been cleared out, and we should *not* consult the previous
// snapshot about any storage values. The only possible alternatives are:
// 1) resurrect happened, and new slot values were set -- those should
// have been handles via pendingStorage above.
// 2) we don't have new values, and can deliver empty response back
if _, destructed := s.db.snapDestructs[s.addrHash]; destructed {
return common.Hash{}
}
enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key[:]))
}
// Otherwise load the value from the database
enc, err := s.getTrie(db).TryGet(key[:])
if err != nil {
s.setError(err)
return common.Hash{}
// If snapshot unavailable or reading from it failed, load from the database
if s.db.snap == nil || err != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageReads += time.Since(start) }(time.Now())
}
if enc, err = s.getTrie(db).TryGet(key[:]); err != nil {
s.setError(err)
return common.Hash{}
}
}
var value common.Hash
if len(enc) > 0 {
@ -283,6 +303,16 @@ func (s *stateObject) updateTrie(db Database) Trie {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now())
}
// Retrieve the snapshot storage map for the object
var storage map[common.Hash][]byte
if s.db.snap != nil {
// Retrieve the old storage map, if available, create a new one otherwise
storage = s.db.snapStorage[s.addrHash]
if storage == nil {
storage = make(map[common.Hash][]byte)
s.db.snapStorage[s.addrHash] = storage
}
}
// Insert all the pending updates into the trie
tr := s.getTrie(db)
for key, value := range s.pendingStorage {
@ -292,13 +322,18 @@ func (s *stateObject) updateTrie(db Database) Trie {
}
s.originStorage[key] = value
var v []byte
if (value == common.Hash{}) {
s.setError(tr.TryDelete(key[:]))
continue
} else {
// Encoding []byte cannot fail, ok to ignore the error.
v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:]))
s.setError(tr.TryUpdate(key[:], v))
}
// If state snapshotting is active, cache the data til commit
if storage != nil {
storage[crypto.Keccak256Hash(key[:])] = v // v will be nil if value is 0x00
}
// Encoding []byte cannot fail, ok to ignore the error.
v, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(value[:]))
s.setError(tr.TryUpdate(key[:], v))
}
if len(s.pendingStorage) > 0 {
s.pendingStorage = make(Storage)

4
core/state/state_test.go
View file

@ -36,7 +36,7 @@ type stateTest struct {
func newStateTest() *stateTest {
db := rawdb.NewMemoryDatabase()
sdb, _ := New(common.Hash{}, NewDatabase(db))
sdb, _ := New(common.Hash{}, NewDatabase(db), nil)
return &stateTest{db: db, state: sdb}
}
@ -146,7 +146,7 @@ func TestSnapshotEmpty(t *testing.T) {
}
func TestSnapshot2(t *testing.T) {
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
stateobjaddr0 := toAddr([]byte("so0"))
stateobjaddr1 := toAddr([]byte("so1"))

152
core/state/statedb.go
View file

@ -25,6 +25,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
@ -66,6 +67,12 @@ type StateDB struct {
db Database
trie Trie
snaps *snapshot.Tree
snap snapshot.Snapshot
snapDestructs map[common.Hash]struct{}
snapAccounts map[common.Hash][]byte
snapStorage map[common.Hash]map[common.Hash][]byte
// This map holds 'live' objects, which will get modified while processing a state transition.
stateObjects map[common.Address]*stateObject
stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie
@ -95,32 +102,44 @@ type StateDB struct {
nextRevisionId int
// Measurements gathered during execution for debugging purposes
AccountReads time.Duration
AccountHashes time.Duration
AccountUpdates time.Duration
AccountCommits time.Duration
StorageReads time.Duration
StorageHashes time.Duration
StorageUpdates time.Duration
StorageCommits time.Duration
AccountReads time.Duration
AccountHashes time.Duration
AccountUpdates time.Duration
AccountCommits time.Duration
StorageReads time.Duration
StorageHashes time.Duration
StorageUpdates time.Duration
StorageCommits time.Duration
SnapshotAccountReads time.Duration
SnapshotStorageReads time.Duration
SnapshotCommits time.Duration
}
// Create a new state from a given trie.
func New(root common.Hash, db Database) (*StateDB, error) {
func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) {
tr, err := db.OpenTrie(root)
if err != nil {
return nil, err
}
return &StateDB{
sdb := &StateDB{
db: db,
trie: tr,
snaps: snaps,
stateObjects: make(map[common.Address]*stateObject),
stateObjectsPending: make(map[common.Address]struct{}),
stateObjectsDirty: make(map[common.Address]struct{}),
logs: make(map[common.Hash][]*types.Log),
preimages: make(map[common.Hash][]byte),
journal: newJournal(),
}, nil
}
if sdb.snaps != nil {
if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil {
sdb.snapDestructs = make(map[common.Hash]struct{})
sdb.snapAccounts = make(map[common.Hash][]byte)
sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte)
}
}
return sdb, nil
}
// setError remembers the first non-nil error it is called with.
@ -152,6 +171,15 @@ func (s *StateDB) Reset(root common.Hash) error {
s.logSize = 0
s.preimages = make(map[common.Hash][]byte)
s.clearJournalAndRefund()
if s.snaps != nil {
s.snapAccounts, s.snapDestructs, s.snapStorage = nil, nil, nil
if s.snap = s.snaps.Snapshot(root); s.snap != nil {
s.snapDestructs = make(map[common.Hash]struct{})
s.snapAccounts = make(map[common.Hash][]byte)
s.snapStorage = make(map[common.Hash]map[common.Hash][]byte)
}
}
return nil
}
@ -438,6 +466,14 @@ func (s *StateDB) updateStateObject(obj *stateObject) {
panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
}
s.setError(s.trie.TryUpdate(addr[:], data))
// If state snapshotting is active, cache the data til commit. Note, this
// update mechanism is not symmetric to the deletion, because whereas it is
// enough to track account updates at commit time, deletions need tracking
// at transaction boundary level to ensure we capture state clearing.
if s.snap != nil {
s.snapAccounts[obj.addrHash] = snapshot.AccountRLP(obj.data.Nonce, obj.data.Balance, obj.data.Root, obj.data.CodeHash)
}
}
// deleteStateObject removes the given object from the state trie.
@ -470,20 +506,44 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
if obj := s.stateObjects[addr]; obj != nil {
return obj
}
// Track the amount of time wasted on loading the object from the database
if metrics.EnabledExpensive {
defer func(start time.Time) { s.AccountReads += time.Since(start) }(time.Now())
// If no live objects are available, attempt to use snapshots
var (
data Account
err error
)
if s.snap != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.SnapshotAccountReads += time.Since(start) }(time.Now())
}
var acc *snapshot.Account
if acc, err = s.snap.Account(crypto.Keccak256Hash(addr[:])); err == nil {
if acc == nil {
return nil
}
data.Nonce, data.Balance, data.CodeHash = acc.Nonce, acc.Balance, acc.CodeHash
if len(data.CodeHash) == 0 {
data.CodeHash = emptyCodeHash
}
data.Root = common.BytesToHash(acc.Root)
if data.Root == (common.Hash{}) {
data.Root = emptyRoot
}
}
}
// Load the object from the database
enc, err := s.trie.TryGet(addr[:])
if len(enc) == 0 {
s.setError(err)
return nil
}
var data Account
if err := rlp.DecodeBytes(enc, &data); err != nil {
log.Error("Failed to decode state object", "addr", addr, "err", err)
return nil
// If snapshot unavailable or reading from it failed, load from the database
if s.snap == nil || err != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.AccountReads += time.Since(start) }(time.Now())
}
enc, err := s.trie.TryGet(addr[:])
if len(enc) == 0 {
s.setError(err)
return nil
}
if err := rlp.DecodeBytes(enc, &data); err != nil {
log.Error("Failed to decode state object", "addr", addr, "err", err)
return nil
}
}
// Insert into the live set
obj := newObject(s, addr, data)
@ -509,12 +569,19 @@ func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that!
var prevdestruct bool
if s.snap != nil && prev != nil {
_, prevdestruct = s.snapDestructs[prev.addrHash]
if !prevdestruct {
s.snapDestructs[prev.addrHash] = struct{}{}
}
}
newobj = newObject(s, addr, Account{})
newobj.setNonce(0) // sets the object to dirty
if prev == nil {
s.journal.append(createObjectChange{account: &addr})
} else {
s.journal.append(resetObjectChange{prev: prev})
s.journal.append(resetObjectChange{prev: prev, prevdestruct: prevdestruct})
}
s.setStateObject(newobj)
return newobj, prev
@ -673,6 +740,16 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
}
if obj.suicided || (deleteEmptyObjects && obj.empty()) {
obj.deleted = true
// If state snapshotting is active, also mark the destruction there.
// Note, we can't do this only at the end of a block because multiple
// transactions within the same block might self destruct and then
// ressurrect an account; but the snapshotter needs both events.
if s.snap != nil {
s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely)
delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect)
delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a ressurrect)
}
} else {
obj.finalise()
}
@ -748,13 +825,14 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
s.stateObjectsDirty = make(map[common.Address]struct{})
}
// Write the account trie changes, measuing the amount of wasted time
var start time.Time
if metrics.EnabledExpensive {
defer func(start time.Time) { s.AccountCommits += time.Since(start) }(time.Now())
start = time.Now()
}
// The onleaf func is called _serially_, so we can reuse the same account
// for unmarshalling every time.
var account Account
return s.trie.Commit(func(leaf []byte, parent common.Hash) error {
root, err := s.trie.Commit(func(leaf []byte, parent common.Hash) error {
if err := rlp.DecodeBytes(leaf, &account); err != nil {
return nil
}
@ -767,4 +845,24 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
}
return nil
})
if metrics.EnabledExpensive {
s.AccountCommits += time.Since(start)
}
// If snapshotting is enabled, update the snapshot tree with this new version
if s.snap != nil {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now())
}
// Only update if there's a state transition (skip empty Clique blocks)
if parent := s.snap.Root(); parent != root {
if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil {
log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err)
}
if err := s.snaps.Cap(root, 127); err != nil { // Persistent layer is 128th, the last available trie
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 127, "err", err)
}
}
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
}
return root, err
}

26
core/state/statedb_test.go
View file

@ -39,7 +39,7 @@ import (
func TestUpdateLeaks(t *testing.T) {
// Create an empty state database
db := rawdb.NewMemoryDatabase()
state, _ := New(common.Hash{}, NewDatabase(db))
state, _ := New(common.Hash{}, NewDatabase(db), nil)
// Update it with some accounts
for i := byte(0); i < 255; i++ {
@ -60,7 +60,7 @@ func TestUpdateLeaks(t *testing.T) {
}
// Ensure that no data was leaked into the database
it := db.NewIterator()
it := db.NewIterator(nil, nil)
for it.Next() {
t.Errorf("State leaked into database: %x -> %x", it.Key(), it.Value())
}
@ -73,8 +73,8 @@ func TestIntermediateLeaks(t *testing.T) {
// Create two state databases, one transitioning to the final state, the other final from the beginning
transDb := rawdb.NewMemoryDatabase()
finalDb := rawdb.NewMemoryDatabase()
transState, _ := New(common.Hash{}, NewDatabase(transDb))
finalState, _ := New(common.Hash{}, NewDatabase(finalDb))
transState, _ := New(common.Hash{}, NewDatabase(transDb), nil)
finalState, _ := New(common.Hash{}, NewDatabase(finalDb), nil)
modify := func(state *StateDB, addr common.Address, i, tweak byte) {
state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@ -118,7 +118,7 @@ func TestIntermediateLeaks(t *testing.T) {
t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex())
}
it := finalDb.NewIterator()
it := finalDb.NewIterator(nil, nil)
for it.Next() {
key, fvalue := it.Key(), it.Value()
tvalue, err := transDb.Get(key)
@ -131,7 +131,7 @@ func TestIntermediateLeaks(t *testing.T) {
}
it.Release()
it = transDb.NewIterator()
it = transDb.NewIterator(nil, nil)
for it.Next() {
key, tvalue := it.Key(), it.Value()
fvalue, err := finalDb.Get(key)
@ -149,7 +149,7 @@ func TestIntermediateLeaks(t *testing.T) {
// https://github.com/ethereum/go-ethereum/pull/15549.
func TestCopy(t *testing.T) {
// Create a random state test to copy and modify "independently"
orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
orig, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
for i := byte(0); i < 255; i++ {
obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@ -385,7 +385,7 @@ func (test *snapshotTest) String() string {
func (test *snapshotTest) run() bool {
// Run all actions and create snapshots.
var (
state, _ = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ = New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
snapshotRevs = make([]int, len(test.snapshots))
sindex = 0
)
@ -399,7 +399,7 @@ func (test *snapshotTest) run() bool {
// Revert all snapshots in reverse order. Each revert must yield a state
// that is equivalent to fresh state with all actions up the snapshot applied.
for sindex--; sindex >= 0; sindex-- {
checkstate, _ := New(common.Hash{}, state.Database())
checkstate, _ := New(common.Hash{}, state.Database(), nil)
for _, action := range test.actions[:test.snapshots[sindex]] {
action.fn(action, checkstate)
}
@ -477,7 +477,7 @@ func TestTouchDelete(t *testing.T) {
// TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy.
// See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512
func TestCopyOfCopy(t *testing.T) {
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
addr := common.HexToAddress("aaaa")
state.SetBalance(addr, big.NewInt(42))
@ -494,7 +494,7 @@ func TestCopyOfCopy(t *testing.T) {
//
// See https://github.com/ethereum/go-ethereum/issues/20106.
func TestCopyCommitCopy(t *testing.T) {
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
// Create an account and check if the retrieved balance is correct
addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe")
@ -566,7 +566,7 @@ func TestCopyCommitCopy(t *testing.T) {
//
// See https://github.com/ethereum/go-ethereum/issues/20106.
func TestCopyCopyCommitCopy(t *testing.T) {
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
// Create an account and check if the retrieved balance is correct
addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe")
@ -656,7 +656,7 @@ func TestCopyCopyCommitCopy(t *testing.T) {
// first, but the journal wiped the entire state object on create-revert.
func TestDeleteCreateRevert(t *testing.T) {
// Create an initial state with a single contract
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
addr := toAddr([]byte("so"))
state.SetBalance(addr, big.NewInt(1))

6
core/state/sync_test.go
View file

@ -41,7 +41,7 @@ type testAccount struct {
func makeTestState() (Database, common.Hash, []*testAccount) {
// Create an empty state
db := NewDatabase(rawdb.NewMemoryDatabase())
state, _ := New(common.Hash{}, db)
state, _ := New(common.Hash{}, db, nil)
// Fill it with some arbitrary data
accounts := []*testAccount{}
@ -72,7 +72,7 @@ func makeTestState() (Database, common.Hash, []*testAccount) {
// account array.
func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
// Check root availability and state contents
state, err := New(root, NewDatabase(db))
state, err := New(root, NewDatabase(db), nil)
if err != nil {
t.Fatalf("failed to create state trie at %x: %v", root, err)
}
@ -113,7 +113,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
if _, err := db.Get(root.Bytes()); err != nil {
return nil // Consider a non existent state consistent.
}
state, err := New(root, NewDatabase(db))
state, err := New(root, NewDatabase(db), nil)
if err != nil {
return err
}

9
core/state_prefetcher.go
View file

@ -54,6 +54,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c
gaspool = new(GasPool).AddGas(block.GasLimit())
)
// Iterate over and process the individual transactions
byzantium := p.config.IsByzantium(block.Number())
for i, tx := range block.Transactions() {
// If block precaching was interrupted, abort
if interrupt != nil && atomic.LoadUint32(interrupt) == 1 {
@ -64,6 +65,14 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c
if err := precacheTransaction(p.config, p.bc, nil, gaspool, statedb, header, tx, cfg); err != nil {
return // Ugh, something went horribly wrong, bail out
}
// If we're pre-byzantium, pre-load trie nodes for the intermediate root
if !byzantium {
statedb.IntermediateRoot(true)
}
}
// If were post-byzantium, pre-load trie nodes for the final root hash
if byzantium {
statedb.IntermediateRoot(true)
}
}

2
core/tx_pool.go
View file

@ -48,7 +48,7 @@ const (
// non-trivial consequences: larger transactions are significantly harder and
// more expensive to propagate; larger transactions also take more resources
// to validate whether they fit into the pool or not.
txMaxSize = 2 * txSlotSize // 64KB, don't bump without EIP-2464 support
txMaxSize = 4 * txSlotSize // 128KB
)
var (

46
core/tx_pool_test.go
View file

@ -86,7 +86,7 @@ func pricedDataTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key
}
func setupTxPool() (*TxPool, *ecdsa.PrivateKey) {
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)}
key, _ := crypto.GenerateKey()
@ -220,7 +220,7 @@ func (c *testChain) State() (*state.StateDB, error) {
// a state change between those fetches.
stdb := c.statedb
if *c.trigger {
c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
c.statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
// simulate that the new head block included tx0 and tx1
c.statedb.SetNonce(c.address, 2)
c.statedb.SetBalance(c.address, new(big.Int).SetUint64(params.Ether))
@ -238,7 +238,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) {
var (
key, _ = crypto.GenerateKey()
address = crypto.PubkeyToAddress(key.PublicKey)
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
trigger = false
)
@ -394,7 +394,7 @@ func TestTransactionChainFork(t *testing.T) {
addr := crypto.PubkeyToAddress(key.PublicKey)
resetState := func() {
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
statedb.AddBalance(addr, big.NewInt(100000000000000))
pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@ -423,7 +423,7 @@ func TestTransactionDoubleNonce(t *testing.T) {
addr := crypto.PubkeyToAddress(key.PublicKey)
resetState := func() {
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
statedb.AddBalance(addr, big.NewInt(100000000000000))
pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)}
@ -614,7 +614,7 @@ func TestTransactionPostponing(t *testing.T) {
t.Parallel()
// Create the pool to test the postponing with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@ -836,7 +836,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) {
t.Parallel()
// Create the pool to test the limit enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -915,8 +915,12 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) {
//
// This logic should not hold for local transactions, unless the local tracking
// mechanism is disabled.
func TestTransactionQueueTimeLimiting(t *testing.T) { testTransactionQueueTimeLimiting(t, false) }
func TestTransactionQueueTimeLimitingNoLocals(t *testing.T) { testTransactionQueueTimeLimiting(t, true) }
func TestTransactionQueueTimeLimiting(t *testing.T) {
testTransactionQueueTimeLimiting(t, false)
}
func TestTransactionQueueTimeLimitingNoLocals(t *testing.T) {
testTransactionQueueTimeLimiting(t, true)
}
func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) {
// Reduce the eviction interval to a testable amount
@ -924,7 +928,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) {
evictionInterval = time.Second
// Create the pool to test the non-expiration enforcement
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1027,7 +1031,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) {
t.Parallel()
// Create the pool to test the limit enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1129,7 +1133,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) {
t.Parallel()
// Create the pool to test the limit enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1163,7 +1167,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) {
t.Parallel()
// Create the pool to test the limit enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1211,7 +1215,7 @@ func TestTransactionPoolRepricing(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@ -1350,7 +1354,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@ -1412,7 +1416,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1533,7 +1537,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1610,7 +1614,7 @@ func TestTransactionDeduplication(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@ -1676,7 +1680,7 @@ func TestTransactionReplacement(t *testing.T) {
t.Parallel()
// Create the pool to test the pricing enforcement with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
@ -1786,7 +1790,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
os.Remove(journal)
// Create the original pool to inject transaction into the journal
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
config := testTxPoolConfig
@ -1884,7 +1888,7 @@ func TestTransactionStatusCheck(t *testing.T) {
t.Parallel()
// Create the pool to test the status retrievals with
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)

10
core/vm/eips.go
View file

@ -60,9 +60,9 @@ func enable1884(jt *JumpTable) {
}
}
func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
balance := interpreter.intPool.get().Set(interpreter.evm.StateDB.GetBalance(contract.Address()))
stack.push(balance)
func opSelfBalance(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) {
balance := interpreter.intPool.get().Set(interpreter.evm.StateDB.GetBalance(callContext.contract.Address()))
callContext.stack.push(balance)
return nil, nil
}
@ -80,9 +80,9 @@ func enable1344(jt *JumpTable) {
}
// opChainID implements CHAINID opcode
func opChainID(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
func opChainID(pc *uint64, interpreter *EVMInterpreter, callContext *callCtx) ([]byte, error) {
chainId := interpreter.intPool.get().Set(interpreter.evm.chainConfig.ChainID)
stack.push(chainId)
callContext.stack.push(chainId)
return nil, nil
}

2
core/vm/gas_table_test.go
View file

@ -81,7 +81,7 @@ func TestEIP2200(t *testing.T) {
for i, tt := range eip2200Tests {
address := common.BytesToAddress([]byte("contract"))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()))
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
statedb.CreateAccount(address)
statedb.SetCode(address, hexutil.MustDecode(tt.input))
statedb.SetState(address, common.Hash{}, common.BytesToHash([]byte{tt.original}))

546
core/vm/instructions.go
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more