forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-17 17:33:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #1 from ethereum/master

Browse source 
Merge with ethereum
This commit is contained in:
saidsyakin 2018-02-21 20:35:22 +04:00 committed by GitHub
commit 3766eca0c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
195 changed files with 4781 additions and 2679 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/CONTRIBUTING.md vendored
View file

@ -2,7 +2,7 @@
Before you do a feature request please check and make sure that it isn't possible
through some other means. The JavaScript enabled console is a powerful feature
in the right hands. Please check our [Bitchin' tricks](https://github.com/ethereum/go-ethereum/wiki/bitchin-tricks) wiki page for more info
in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
and help.
## Contributing

14
.mailmap
View file

@ -65,7 +65,8 @@ Enrique Fynn <enriquefynn@gmail.com>
Vincent G <caktux@gmail.com>
RJ Catalano <rj@erisindustries.com>
RJ Catalano <catalanor0220@gmail.com>
RJ Catalano <catalanor0220@gmail.com> <rj@erisindustries.com>
Nchinda Nchinda <nchinda2@gmail.com>
@ -109,3 +110,14 @@ Frank Wang <eternnoir@gmail.com>
Gary Rong <garyrong0905@gmail.com>
Guillaume Nicolas <guin56@gmail.com>
Sorin Neacsu <sorin.neacsu@gmail.com>
Sorin Neacsu <sorin.neacsu@gmail.com> <sorin@users.noreply.github.com>
Valentin Wüstholz <wuestholz@gmail.com>
Valentin Wüstholz <wuestholz@gmail.com> <wuestholz@users.noreply.github.com>
Armin Braun <me@obrown.io>
Ernesto del Toro <ernesto.deltoro@gmail.com>
Ernesto del Toro <ernesto.deltoro@gmail.com> <ernestodeltoro@users.noreply.github.com>

2
.travis.yml
View file

@ -185,6 +185,8 @@ matrix:
- xctool -version
- xcrun simctl list
# Workaround for https://github.com/golang/go/issues/23749
- export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
# This builder does the Azure archive purges to avoid accumulating junk

92
AUTHORS
View file

@ -1,85 +1,173 @@
# This is the official list of go-ethereum authors for copyright purposes.
Afri Schoedon <5chdn@users.noreply.github.com>
Agustin Armellini Fischer <armellini13@gmail.com>
Airead <fgh1987168@gmail.com>
Alan Chen <alanchchen@users.noreply.github.com>
Alejandro Isaza <alejandro.isaza@gmail.com>
Ales Katona <ales@coinbase.com>
Alex Leverington <alex@ethdev.com>
Alex Wu <wuyiding@gmail.com>
Alexandre Van de Sande <alex.vandesande@ethdev.com>
Ali Hajimirza <Ali92hm@users.noreply.github.com>
Anton Evangelatov <anton.evangelatov@gmail.com>
Arba Sasmoyo <arba.sasmoyo@gmail.com>
Armani Ferrante <armaniferrante@berkeley.edu>
Armin Braun <me@obrown.io>
Aron Fischer <github@aron.guru>
Bas van Kervel <bas@ethdev.com>
Benjamin Brent <benjamin@benjaminbrent.com>
Benoit Verkindt <benoit.verkindt@gmail.com>
Bo <bohende@gmail.com>
Bo Ye <boy.e.computer.1982@outlook.com>
Bob Glickstein <bobg@users.noreply.github.com>
Brian Schroeder <bts@gmail.com>
Casey Detrio <cdetrio@gmail.com>
Chase Wright <mysticryuujin@gmail.com>
Christoph Jentzsch <jentzsch.software@gmail.com>
Daniel A. Nagy <nagy.da@gmail.com>
Daniel Sloof <goapsychadelic@gmail.com>
Darrel Herbst <dherbst@gmail.com>
Dave Appleton <calistralabs@gmail.com>
Diego Siqueira <DiSiqueira@users.noreply.github.com>
Dmitry Shulyak <yashulyak@gmail.com>
Egon Elbre <egonelbre@gmail.com>
Elias Naur <elias.naur@gmail.com>
Elliot Shepherd <elliot@identitii.com>
Enrique Fynn <enriquefynn@gmail.com>
Ernesto del Toro <ernesto.deltoro@gmail.com>
Ethan Buchman <ethan@coinculture.info>
Eugene Valeyev <evgen.povt@gmail.com>
Evangelos Pappas <epappas@evalonlabs.com>
Evgeny Danilenko <6655321@bk.ru>
Fabian Vogelsteller <fabian@frozeman.de>
Fabio Barone <fabio.barone.co@gmail.com>
Fabio Berger <fabioberger1991@gmail.com>
FaceHo <facehoshi@gmail.com>
Felix Lange <fjl@twurst.com>
Fiisio <liangcszzu@163.com>
Frank Wang <eternnoir@gmail.com>
Furkan KAMACI <furkankamaci@gmail.com>
Gary Rong <garyrong0905@gmail.com>
George Ornbo <george@shapeshed.com>
Gregg Dourgarian <greggd@tempworks.com>
Guillaume Ballet <gballet@gmail.com>
Guillaume Nicolas <guin56@gmail.com>
Gustav Simonsson <gustav.simonsson@gmail.com>
Hao Bryan Cheng <haobcheng@gmail.com>
Henning Diedrich <hd@eonblast.com>
Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
Ivan Daniluk <ivan.daniluk@gmail.com>
Jae Kwon <jkwon.work@gmail.com>
Jamie Pitts <james.pitts@gmail.com>
Janoš Guljaš <janos@users.noreply.github.com>
Jason Carver <jacarver@linkedin.com>
Jay Guo <guojiannan1101@gmail.com>
Jeff R. Allen <jra@nella.org>
Jeffrey Wilcke <jeffrey@ethereum.org>
Jens Agerberg <github@agerberg.me>
Jia Chenhui <jiachenhui1989@gmail.com>
Jim McDonald <Jim@mcdee.net>
Joel Burget <joelburget@gmail.com>
Jonathan Brown <jbrown@bluedroplet.com>
Joseph Chow <ethereum@outlook.com>
Justin Clark-Casey <justincc@justincc.org>
Justin Drake <drakefjustin@gmail.com>
Kenji Siu <kenji@isuntv.com>
Kobi Gurkan <kobigurk@gmail.com>
Konrad Feldmeier <konrad@brainbot.com>
Kurkó Mihály <kurkomisi@users.noreply.github.com>
Kyuntae Ethan Kim <ethan.kyuntae.kim@gmail.com>
Lefteris Karapetsas <lefteris@refu.co>
Leif Jurvetson <leijurv@gmail.com>
Leo Shklovskii <leo@thermopylae.net>
Lewis Marshall <lewis@lmars.net>
Lio李欧 <lionello@users.noreply.github.com>
Louis Holbrook <dev@holbrook.no>
Luca Zeug <luclu@users.noreply.github.com>
Magicking <s@6120.eu>
Maran Hidskes <maran.hidskes@gmail.com>
Marek Kotewicz <marek.kotewicz@gmail.com>
Mark <markya0616@gmail.com>
Martin Holst Swende <martin@swende.se>
Matthew Di Ferrante <mattdf@users.noreply.github.com>
Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
Maximilian Meister <mmeister@suse.de>
Micah Zoltu <micah@zoltu.net>
Michael Ruminer <michael.ruminer+github@gmail.com>
Miguel Mota <miguelmota2@gmail.com>
Miya Chen <miyatlchen@gmail.com>
Nchinda Nchinda <nchinda2@gmail.com>
Nick Dodson <silentcicero@outlook.com>
Nick Johnson <arachnid@notdot.net>
Nicolas Guillaume <gunicolas@sqli.com>
Noman <noman@noman.land>
Oli Bye <olibye@users.noreply.github.com>
Paul Litvak <litvakpol@012.net.il>
Paulo L F Casaretto <pcasaretto@gmail.com>
Paweł Bylica <chfast@gmail.com>
Peter Pratscher <pratscher@gmail.com>
Petr Mikusek <petr@mikusek.info>
Péter Szilágyi <peterke@gmail.com>
RJ Catalano <rj@erisindustries.com>
RJ Catalano <catalanor0220@gmail.com>
Ramesh Nair <ram@hiddentao.com>
Ricardo Catalinas Jiménez <r@untroubled.be>
Ricardo Domingos <ricardohsd@gmail.com>
Richard Hart <richardhart92@gmail.com>
Rob <robert@rojotek.com>
Robert Zaremba <robert.zaremba@scale-it.pl>
Russ Cox <rsc@golang.org>
Rémy Roy <remyroy@remyroy.com>
S. Matthew English <s-matthew-english@users.noreply.github.com>
Shintaro Kaneko <kaneshin0120@gmail.com>
Sorin Neacsu <sorin.neacsu@gmail.com>
Stein Dekker <dekker.stein@gmail.com>
Steve Waldman <swaldman@mchange.com>
Steven Roose <stevenroose@gmail.com>
Taylor Gerring <taylor.gerring@gmail.com>
Thomas Bocek <tom@tomp2p.net>
Ti Zhou <tizhou1986@gmail.com>
Tosh Camille <tochecamille@gmail.com>
Valentin Wüstholz <wuestholz@users.noreply.github.com>
Valentin Wüstholz <wuestholz@gmail.com>
Victor Farazdagi <simple.square@gmail.com>
Victor Tran <vu.tran54@gmail.com>
Viktor Trón <viktor.tron@gmail.com>
Ville Sundell <github@solarius.fi>
Vincent G <caktux@gmail.com>
Vitalik Buterin <v@buterin.com>
Vitaly V <vvelikodny@gmail.com>
Vivek Anand <vivekanand1101@users.noreply.github.com>
Vlad Gluhovsky <gluk256@users.noreply.github.com>
Yohann Léon <sybiload@gmail.com>
Yoichi Hirai <i@yoichihirai.com>
Yondon Fu <yondon.fu@gmail.com>
Zach <zach.ramsay@gmail.com>
Zahoor Mohamed <zahoor@zahoor.in>
Zoe Nolan <github@zoenolan.org>
Zsolt Felföldi <zsfelfoldi@gmail.com>
am2rican5 <am2rican5@gmail.com>
ayeowch <ayeowch@gmail.com>
b00ris <b00ris@mail.ru>
bailantaotao <Edwin@maicoin.com>
baizhenxuan <nkbai@163.com>
bloonfield <bloonfield@163.com>
changhong <changhong.yu@shanbay.com>
evgk <evgeniy.kamyshev@gmail.com>
ferhat elmas <elmas.ferhat@gmail.com>
holisticode <holistic.computing@gmail.com>
jtakalai <juuso.takalainen@streamr.com>
ken10100147 <sunhongping@kanjian.com>
ligi <ligi@ligi.de>
mark.lin <mark@maicoin.com>
necaremus <necaremus@gmail.com>
njupt-moon <1015041018@njupt.edu.cn>
nkbai <nkbai@163.com>
rhaps107 <dod-source@yandex.ru>
slumber1122 <slumber1122@gmail.com>
sunxiaojun2014 <sunxiaojun-xy@360.cn>
terasum <terasum@163.com>
tsarpaul <Litvakpol@012.net.il>
xiekeyang <xiekeyang@users.noreply.github.com>
yoza <yoza.is12s@gmail.com>
ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
Максим Чусовлянов <mchusovlianov@gmail.com>

30
README.md
View file

@ -5,6 +5,7 @@ Official golang implementation of the Ethereum protocol.
[![API Reference](
https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
)](https://godoc.org/github.com/ethereum/go-ethereum)
[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
Automated builds are available for stable releases and the unstable master branch.
@ -56,16 +57,14 @@ the user doesn't care about years-old historical data, so we can fast-sync quick
state of the network. To do so:
```
$ geth --fast --cache=512 console
$ geth console
```
This command will:
* Start geth in fast sync mode (`--fast`), causing it to download more data in exchange for avoiding
processing the entire history of the Ethereum network, which is very CPU intensive.
* Bump the memory allowance of the database to 512MB (`--cache=512`), which can help significantly in
sync times especially for HDD users. This flag is optional and you can set it as high or as low as
you'd like, though we'd recommend the 512MB - 2GB range.
* Start geth in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to
download more data in exchange for avoiding processing the entire history of the Ethereum network,
which is very CPU intensive.
* Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
(via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
@ -80,12 +79,11 @@ entire system. In other words, instead of attaching to the main network, you wan
network with your node, which is fully equivalent to the main network, but with play-Ether only.
```
$ geth --testnet --fast --cache=512 console
$ geth --testnet console
```
The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they
are equally useful on the testnet too. Please see above for their explanations if you've skipped to
here.
The `console` subcommand have the exact same meaning as above and they are equally useful on the
testnet too. Please see above for their explanations if you've skipped to here.
Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
@ -102,6 +100,14 @@ over between the main network and test network, you should make sure to always u
for play-money and real-money. Unless you manually move accounts, Geth will by default correctly
separate the two networks and will not make any accounts available between them.*
### Full node on the Rinkeby test network
The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
```
$ geth --rinkeby console
```
### Configuration
As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via:
@ -125,10 +131,10 @@ One of the quickest ways to get Ethereum up and running on your machine is by us
```
docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
-p 8545:8545 -p 30303:30303 \
ethereum/client-go --fast --cache=512
ethereum/client-go
```
This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
This will start geth in fast-sync mode with a DB memory allowance of 1GB just as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.

2
VERSION
View file

@ -1 +1 @@
1.8.0
1.8.2

6
accounts/abi/abi.go
View file

@ -136,11 +136,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
// MethodById looks up a method by the 4-byte id
// returns nil if none found
func (abi *ABI) MethodById(sigdata []byte) *Method {
func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
for _, method := range abi.Methods {
if bytes.Equal(method.Id(), sigdata[:4]) {
return &method
return &method, nil
}
}
return nil
return nil, fmt.Errorf("no method with id: %#x", sigdata[:4])
}

6
accounts/abi/abi_test.go
View file

@ -689,7 +689,11 @@ func TestABI_MethodById(t *testing.T) {
}
for name, m := range abi.Methods {
a := fmt.Sprintf("%v", m)
b := fmt.Sprintf("%v", abi.MethodById(m.Id()))
m2, err := abi.MethodById(m.Id())
if err != nil {
t.Fatalf("Failed to look up ABI method: %v", err)
}
b := fmt.Sprintf("%v", m2)
if a != b {
t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
}

116
accounts/abi/argument.go
View file

@ -67,6 +67,17 @@ func (arguments Arguments) LengthNonIndexed() int {
return out
}
// NonIndexed returns the arguments with indexed arguments filtered out
func (arguments Arguments) NonIndexed() Arguments {
var ret []Argument
for _, arg := range arguments {
if !arg.Indexed {
ret = append(ret, arg)
}
}
return ret
}
// isTuple returns true for non-atomic constructs, like (uint,uint) or uint[]
func (arguments Arguments) isTuple() bool {
return len(arguments) > 1
@ -74,21 +85,25 @@ func (arguments Arguments) isTuple() bool {
// Unpack performs the operation hexdata -> Go format
func (arguments Arguments) Unpack(v interface{}, data []byte) error {
if arguments.isTuple() {
return arguments.unpackTuple(v, data)
}
return arguments.unpackAtomic(v, data)
}
func (arguments Arguments) unpackTuple(v interface{}, output []byte) error {
// make sure the passed value is arguments pointer
valueOf := reflect.ValueOf(v)
if reflect.Ptr != valueOf.Kind() {
if reflect.Ptr != reflect.ValueOf(v).Kind() {
return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
}
marshalledValues, err := arguments.UnpackValues(data)
if err != nil {
return err
}
if arguments.isTuple() {
return arguments.unpackTuple(v, marshalledValues)
}
return arguments.unpackAtomic(v, marshalledValues)
}
func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error {
var (
value = valueOf.Elem()
value = reflect.ValueOf(v).Elem()
typ = value.Type()
kind = value.Kind()
)
@ -110,30 +125,9 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error {
exists[field] = true
}
}
// `i` counts the nonindexed arguments.
// `j` counts the number of complex types.
// both `i` and `j` are used to to correctly compute `data` offset.
for i, arg := range arguments.NonIndexed() {
i, j := -1, 0
for _, arg := range arguments {
if arg.Indexed {
// can't read, continue
continue
}
i++
marshalledValue, err := toGoType((i+j)*32, arg.Type, output)
if err != nil {
return err
}
if arg.Type.T == ArrayTy {
// combined index ('i' + 'j') need to be adjusted only by size of array, thus
// we need to decrement 'j' because 'i' was incremented
j += arg.Type.Size - 1
}
reflectValue := reflect.ValueOf(marshalledValue)
reflectValue := reflect.ValueOf(marshalledValues[i])
switch kind {
case reflect.Struct:
@ -166,34 +160,52 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error {
}
// unpackAtomic unpacks ( hexdata -> go ) a single value
func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error {
// make sure the passed value is arguments pointer
valueOf := reflect.ValueOf(v)
if reflect.Ptr != valueOf.Kind() {
return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error {
if len(marshalledValues) != 1 {
return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues))
}
arg := arguments[0]
if arg.Indexed {
return fmt.Errorf("abi: attempting to unpack indexed variable into element.")
}
value := valueOf.Elem()
marshalledValue, err := toGoType(0, arg.Type, output)
if err != nil {
return err
}
return set(value, reflect.ValueOf(marshalledValue), arg)
elem := reflect.ValueOf(v).Elem()
reflectValue := reflect.ValueOf(marshalledValues[0])
return set(elem, reflectValue, arguments.NonIndexed()[0])
}
// Unpack performs the operation Go format -> Hexdata
// UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification,
// without supplying a struct to unpack into. Instead, this method returns a list containing the
// values. An atomic argument will be a list with one element.
func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
retval := make([]interface{}, 0, arguments.LengthNonIndexed())
virtualArgs := 0
for index, arg := range arguments.NonIndexed() {
marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
if arg.Type.T == ArrayTy {
// If we have a static array, like [3]uint256, these are coded as
// just like uint256,uint256,uint256.
// This means that we need to add two 'virtual' arguments when
// we count the index from now on
virtualArgs += arg.Type.Size - 1
}
if err != nil {
return nil, err
}
retval = append(retval, marshalledValue)
}
return retval, nil
}
// PackValues performs the operation Go format -> Hexdata
// It is the semantic opposite of UnpackValues
func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) {
return arguments.Pack(args...)
}
// Pack performs the operation Go format -> Hexdata
func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
// Make sure arguments match up and pack them
abiArgs := arguments
if len(args) != len(abiArgs) {
return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(abiArgs))
}
// variable input is the output appended at the end of packed
// output. This is used for strings and bytes types input.
var variableInput []byte
@ -207,7 +219,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
inputOffset += 32
}
}
var ret []byte
for i, a := range args {
input := abiArgs[i]
@ -216,7 +227,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
if err != nil {
return nil, err
}
// check for a slice type (string, bytes, slice)
if input.Type.requiresLengthPrefix() {
// calculate the offset

14
accounts/abi/bind/backends/simulated.go
View file

@ -68,7 +68,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
database, _ := ethdb.NewMemDatabase()
genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
genesis.MustCommit(database)
blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
backend := &SimulatedBackend{
database: database,
@ -102,8 +102,10 @@ func (b *SimulatedBackend) Rollback() {
func (b *SimulatedBackend) rollback() {
blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
}
// CodeAt returns the code associated with a certain account in the blockchain.
@ -309,8 +311,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
}
block.AddTx(tx)
})
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
return nil
}
@ -386,8 +390,10 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
}
block.OffsetTime(int64(adjustment.Seconds()))
})
statedb, _ := b.blockchain.State()
b.pendingBlock = blocks[0]
b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
return nil
}

2
accounts/abi/pack_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2015 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify

41
accounts/abi/unpack.go
View file

@ -1,4 +1,4 @@
// Copyright 2015 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@ -95,6 +95,9 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) {
// iteratively unpack elements
func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
if size < 0 {
return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size)
}
if start+32*size > len(output) {
return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
}
@ -181,16 +184,32 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
// interprets a 32 byte slice as an offset and then determines which indice to look to decode the type.
func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) {
offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
if offset+32 > len(output) {
return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
}
length = int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
if offset+32+length > len(output) {
return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length)
}
start = offset + 32
bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32])
bigOffsetEnd.Add(bigOffsetEnd, common.Big32)
outputLength := big.NewInt(int64(len(output)))
//fmt.Printf("LENGTH PREFIX INFO: \nsize: %v\noffset: %v\nstart: %v\n", length, offset, start)
if bigOffsetEnd.Cmp(outputLength) > 0 {
return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", bigOffsetEnd, outputLength)
}
if bigOffsetEnd.BitLen() > 63 {
return 0, 0, fmt.Errorf("abi offset larger than int64: %v", bigOffsetEnd)
}
offsetEnd := int(bigOffsetEnd.Uint64())
lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd])
totalSize := big.NewInt(0)
totalSize.Add(totalSize, bigOffsetEnd)
totalSize.Add(totalSize, lengthBig)
if totalSize.BitLen() > 63 {
return 0, 0, fmt.Errorf("abi length larger than int64: %v", totalSize)
}
if totalSize.Cmp(outputLength) > 0 {
return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %v require %v", outputLength, totalSize)
}
start = int(bigOffsetEnd.Uint64())
length = int(lengthBig.Uint64())
return
}

74
accounts/abi/unpack_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2015 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@ -130,7 +130,7 @@ var unpackTests = []unpackTest{
{
def: `[{"type": "bytes32"}]`,
enc: "0100000000000000000000000000000000000000000000000000000000000000",
want: common.HexToHash("0100000000000000000000000000000000000000000000000000000000000000"),
want: [32]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
},
{
def: `[{"type": "function"}]`,
@ -683,3 +683,73 @@ func TestUnmarshal(t *testing.T) {
t.Fatal("expected error:", err)
}
}
func TestOOMMaliciousInput(t *testing.T) {
oomTests := []unpackTest{
{
def: `[{"type": "uint8[]"}]`,
enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset
"0000000000000000000000000000000000000000000000000000000000000003" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Length larger than 64 bits
def: `[{"type": "uint8[]"}]`,
enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset
"00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Offset very large (over 64 bits)
def: `[{"type": "uint8[]"}]`,
enc: "00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000020" + // offset
"0000000000000000000000000000000000000000000000000000000000000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Offset very large (below 64 bits)
def: `[{"type": "uint8[]"}]`,
enc: "0000000000000000000000000000000000000000000000007ffffffffff00020" + // offset
"0000000000000000000000000000000000000000000000000000000000000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Offset negative (as 64 bit)
def: `[{"type": "uint8[]"}]`,
enc: "000000000000000000000000000000000000000000000000f000000000000020" + // offset
"0000000000000000000000000000000000000000000000000000000000000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Negative length
def: `[{"type": "uint8[]"}]`,
enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset
"000000000000000000000000000000000000000000000000f000000000000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
{ // Very large length
def: `[{"type": "uint8[]"}]`,
enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset
"0000000000000000000000000000000000000000000000007fffffffff000002" + // num elems
"0000000000000000000000000000000000000000000000000000000000000001" + // elem 1
"0000000000000000000000000000000000000000000000000000000000000002", // elem 2
},
}
for i, test := range oomTests {
def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
abi, err := JSON(strings.NewReader(def))
if err != nil {
t.Fatalf("invalid ABI definition %s: %v", def, err)
}
encb, err := hex.DecodeString(test.enc)
if err != nil {
t.Fatalf("invalid hex: %s" + test.enc)
}
_, err = abi.Methods["method"].Outputs.UnpackValues(encb)
if err == nil {
t.Fatalf("Expected error on malicious input, test %d", i)
}
}
}

2
accounts/errors.go
View file

@ -62,7 +62,7 @@ func NewAuthNeededError(needed string) error {
}
}
// Error implements the standard error interfacel.
// Error implements the standard error interface.
func (err *AuthNeededError) Error() string {
return fmt.Sprintf("authentication needed: %s", err.Needed)
}

3
build/ci.go
View file

@ -121,7 +121,8 @@ var (
// Note: vivid is unsupported because there is no golang-1.6 package for it.
// Note: wily is unsupported because it was officially deprecated on lanchpad.
// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "zesty", "artful"}
// Note: zesty is unsupported because it was officially deprecated on lanchpad.
debDistros = []string{"trusty", "xenial", "artful", "bionic"}
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))

5
build/update-license.go
View file

@ -55,10 +55,9 @@ var (
"crypto/sha3/",
"internal/jsre/deps",
"log/",
"common/bitutil/bitutil",
// don't license generated files
"contracts/chequebook/contract/",
"contracts/ens/contract/",
"contracts/release/contract.go",
"contracts/chequebook/contract/code.go",
}
// paths with this prefix are licensed as GPL. all other files are LGPL.

7
cmd/bootnode/main.go
View file

@ -122,7 +122,12 @@ func main() {
utils.Fatalf("%v", err)
}
} else {
if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil {
cfg := discover.Config{
PrivateKey: nodeKey,
AnnounceAddr: realaddr,
NetRestrict: restrictList,
}
if _, err := discover.ListenUDP(conn, cfg); err != nil {
utils.Fatalf("%v", err)
}
}

16
cmd/ethkey/inspect.go
View file

@ -1,3 +1,19 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (

2
cmd/ethkey/message_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 The go-ethereum Authors
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify

2
cmd/ethkey/run_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 The go-ethereum Authors
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify

14
cmd/evm/json_logger.go
View file

@ -1,18 +1,18 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
// This file is part of go-ethereum.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main

4
cmd/evm/runner.go
View file

@ -96,7 +96,9 @@ func runCmd(ctx *cli.Context) error {
}
if ctx.GlobalString(GenesisFlag.Name) != "" {
gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
_, statedb = gen.ToBlock()
db, _ := ethdb.NewMemDatabase()
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
chainConfig = gen.Config
} else {
db, _ := ethdb.NewMemDatabase()

7
cmd/geth/chaincmd.go
View file

@ -67,6 +67,9 @@ It expects the genesis file as argument.`,
utils.DataDirFlag,
utils.CacheFlag,
utils.LightModeFlag,
utils.GCModeFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -202,7 +205,7 @@ func importChain(ctx *cli.Context) error {
if len(ctx.Args()) == 1 {
if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
utils.Fatalf("Import error: %v", err)
log.Error("Import error", "err", err)
}
} else {
for _, arg := range ctx.Args() {
@ -211,7 +214,7 @@ func importChain(ctx *cli.Context) error {
}
}
}
chain.Stop()
fmt.Printf("Import done in %v.\n\n", time.Since(start))
// Output pre-compaction stats mostly to see the import trashing

3
cmd/geth/consolecmd.go
View file

@ -22,6 +22,7 @@ import (
"os/signal"
"path/filepath"
"strings"
"syscall"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/console"
@ -207,7 +208,7 @@ func ephemeralConsole(ctx *cli.Context) error {
}
// Wait for pending callbacks, but stop for Ctrl-C.
abort := make(chan os.Signal, 1)
signal.Notify(abort, os.Interrupt)
signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-abort

4
cmd/geth/main.go
View file

@ -85,10 +85,13 @@ var (
utils.FastSyncFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.LightServFlag,
utils.LightPeersFlag,
utils.LightKDFFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,
@ -111,6 +114,7 @@ var (
utils.VMEnableDebugFlag,
utils.NetworkIdFlag,
utils.RPCCORSDomainFlag,
utils.RPCVirtualHostsFlag,
utils.EthStatsURLFlag,
utils.MetricsEnabledFlag,
utils.FakePoWFlag,

7
cmd/geth/usage.go
View file

@ -22,10 +22,11 @@ import (
"io"
"sort"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/internal/debug"
"gopkg.in/urfave/cli.v1"
"strings"
)
// AppHelpTemplate is the test template for the default, global app help topic.
@ -74,6 +75,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.TestnetFlag,
utils.RinkebyFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.EthStatsURLFlag,
utils.IdentityFlag,
utils.LightServFlag,
@ -127,6 +129,8 @@ var AppHelpFlagGroups = []flagGroup{
Name: "PERFORMANCE TUNING",
Flags: []cli.Flag{
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
},
},
@ -152,6 +156,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.IPCDisabledFlag,
utils.IPCPathFlag,
utils.RPCCORSDomainFlag,
utils.RPCVirtualHostsFlag,
utils.JSpathFlag,
utils.ExecFlag,
utils.PreloadJSFlag,

16
cmd/p2psim/main.go
View file

@ -1,3 +1,19 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// p2psim provides a command-line client for a simulation HTTP API.
//
// Here is an example of creating a 2 node network with the first node

78
cmd/puppeth/module_dashboard.go
View file

@ -117,7 +117,7 @@ var dashboardContent = `
<br/>
<p>To run an archive node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=1024 --syncmode=full{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}</pre>
</p>
<br/>
<p>You can download Geth from <a href="https://geth.ethereum.org/downloads/" target="about:blank">https://geth.ethereum.org/downloads/</a>.</p>
@ -136,7 +136,7 @@ var dashboardContent = `
<br/>
<p>To run a full node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFullFlat}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=512{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}</pre>
</p>
<br/>
<p>You can download Geth from <a href="https://geth.ethereum.org/downloads/" target="about:blank">https://geth.ethereum.org/downloads/</a>.</p>
@ -158,7 +158,7 @@ var dashboardContent = `
<br/>
<p>To run a light node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}</pre>
</p>
<br/>
<p>You can download Geth from <a href="https://geth.ethereum.org/downloads/" target="about:blank">https://geth.ethereum.org/downloads/</a>.</p>
@ -177,7 +177,7 @@ var dashboardContent = `
<br/>
<p>To run an embedded node, download <a href="/{{.GethGenesis}}"><code>{{.GethGenesis}}</code></a> and start Geth with:
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesLightFlat}}</pre>
<pre>geth --networkid={{.NetworkID}} --datadir=$HOME/.{{.Network}} --cache=16 --ethash.cachesinmem=1 --syncmode=light{{if .Ethstats}} --ethstats='{{.Ethstats}}'{{end}} --bootnodes={{.BootnodesFlat}}</pre>
</p>
<br/>
<p>You can download Geth from <a href="https://geth.ethereum.org/downloads/" target="about:blank">https://geth.ethereum.org/downloads/</a>.</p>
@ -208,7 +208,7 @@ var dashboardContent = `
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
</p>
<p>With your local chain initialized, you can start the Ethereum Wallet:
<pre>ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}</pre>
<pre>ethereumwallet --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}</pre>
<p>
<br/>
<p>You can download the Ethereum Wallet from <a href="https://github.com/ethereum/mist/releases" target="about:blank">https://github.com/ethereum/mist/releases</a>.</p>
@ -229,7 +229,7 @@ var dashboardContent = `
<pre>geth --datadir=$HOME/.{{.Network}} init {{.GethGenesis}}</pre>
</p>
<p>With your local chain initialized, you can start Mist:
<pre>mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFullFlat}}</pre>
<pre>mist --rpc $HOME/.{{.Network}}/geth.ipc --node-networkid={{.NetworkID}} --node-datadir=$HOME/.{{.Network}}{{if .Ethstats}} --node-ethstats='{{.Ethstats}}'{{end}} --node-bootnodes={{.BootnodesFlat}}</pre>
<p>
<br/>
<p>You can download the Mist browser from <a href="https://github.com/ethereum/mist/releases" target="about:blank">https://github.com/ethereum/mist/releases</a>.</p>
@ -261,7 +261,7 @@ var dashboardContent = `
<p>Inside your Java code you can now import the geth archive and connect to Ethereum:
<pre>import org.ethereum.geth.*;</pre>
<pre>
Enodes bootnodes = new Enodes();{{range .BootnodesLight}}
Enodes bootnodes = new Enodes();{{range .Bootnodes}}
bootnodes.append(new Enode("{{.}}"));{{end}}
NodeConfig config = new NodeConfig();
@ -294,7 +294,7 @@ node.start();
<pre>
var error: NSError?
let bootnodes = GethNewEnodesEmpty(){{range .BootnodesLight}}
let bootnodes = GethNewEnodesEmpty(){{range .Bootnodes}}
bootnodes?.append(GethNewEnode("{{.}}", &error)){{end}}
let config = GethNewNodeConfig()
@ -595,44 +595,42 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
statsLogin = ""
}
indexfile := new(bytes.Buffer)
bootCpp := make([]string, len(conf.bootFull))
for i, boot := range conf.bootFull {
bootCpp := make([]string, len(conf.bootnodes))
for i, boot := range conf.bootnodes {
bootCpp[i] = "required:" + strings.TrimPrefix(boot, "enode://")
}
bootHarmony := make([]string, len(conf.bootFull))
for i, boot := range conf.bootFull {
bootHarmony := make([]string, len(conf.bootnodes))
for i, boot := range conf.bootnodes {
bootHarmony[i] = fmt.Sprintf("-Dpeer.active.%d.url=%s", i, boot)
}
bootPython := make([]string, len(conf.bootFull))
for i, boot := range conf.bootFull {
bootPython := make([]string, len(conf.bootnodes))
for i, boot := range conf.bootnodes {
bootPython[i] = "'" + boot + "'"
}
template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{
"Network": network,
"NetworkID": conf.Genesis.Config.ChainId,
"NetworkTitle": strings.Title(network),
"EthstatsPage": config.ethstats,
"ExplorerPage": config.explorer,
"WalletPage": config.wallet,
"FaucetPage": config.faucet,
"GethGenesis": network + ".json",
"BootnodesFull": conf.bootFull,
"BootnodesLight": conf.bootLight,
"BootnodesFullFlat": strings.Join(conf.bootFull, ","),
"BootnodesLightFlat": strings.Join(conf.bootLight, ","),
"Ethstats": statsLogin,
"Ethash": conf.Genesis.Config.Ethash != nil,
"CppGenesis": network + "-cpp.json",
"CppBootnodes": strings.Join(bootCpp, " "),
"HarmonyGenesis": network + "-harmony.json",
"HarmonyBootnodes": strings.Join(bootHarmony, " "),
"ParityGenesis": network + "-parity.json",
"PythonGenesis": network + "-python.json",
"PythonBootnodes": strings.Join(bootPython, ","),
"Homestead": conf.Genesis.Config.HomesteadBlock,
"Tangerine": conf.Genesis.Config.EIP150Block,
"Spurious": conf.Genesis.Config.EIP155Block,
"Byzantium": conf.Genesis.Config.ByzantiumBlock,
"Network": network,
"NetworkID": conf.Genesis.Config.ChainId,
"NetworkTitle": strings.Title(network),
"EthstatsPage": config.ethstats,
"ExplorerPage": config.explorer,
"WalletPage": config.wallet,
"FaucetPage": config.faucet,
"GethGenesis": network + ".json",
"Bootnodes": conf.bootnodes,
"BootnodesFlat": strings.Join(conf.bootnodes, ","),
"Ethstats": statsLogin,
"Ethash": conf.Genesis.Config.Ethash != nil,
"CppGenesis": network + "-cpp.json",
"CppBootnodes": strings.Join(bootCpp, " "),
"HarmonyGenesis": network + "-harmony.json",
"HarmonyBootnodes": strings.Join(bootHarmony, " "),
"ParityGenesis": network + "-parity.json",
"PythonGenesis": network + "-python.json",
"PythonBootnodes": strings.Join(bootPython, ","),
"Homestead": conf.Genesis.Config.HomesteadBlock,
"Tangerine": conf.Genesis.Config.EIP150Block,
"Spurious": conf.Genesis.Config.EIP155Block,
"Byzantium": conf.Genesis.Config.ByzantiumBlock,
})
files[filepath.Join(workdir, "index.html")] = indexfile.Bytes()
@ -651,7 +649,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
harmonySpecJSON, _ := conf.Genesis.MarshalJSON()
files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON
paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootFull)
paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes)
if err != nil {
return nil, err
}

8
cmd/puppeth/module_faucet.go
View file

@ -93,7 +93,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
"NetworkID": config.node.network,
"Bootnodes": strings.Join(bootnodes, ","),
"Ethstats": config.node.ethstats,
"EthPort": config.node.portFull,
"EthPort": config.node.port,
"CaptchaToken": config.captchaToken,
"CaptchaSecret": config.captchaSecret,
"FaucetName": strings.Title(network),
@ -110,7 +110,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
"Datadir": config.node.datadir,
"VHost": config.host,
"ApiPort": config.port,
"EthPort": config.node.portFull,
"EthPort": config.node.port,
"EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
"CaptchaToken": config.captchaToken,
"CaptchaSecret": config.captchaSecret,
@ -158,7 +158,7 @@ func (info *faucetInfos) Report() map[string]string {
report := map[string]string{
"Website address": info.host,
"Website listener port": strconv.Itoa(info.port),
"Ethereum listener port": strconv.Itoa(info.node.portFull),
"Ethereum listener port": strconv.Itoa(info.node.port),
"Funding amount (base tier)": fmt.Sprintf("%d Ethers", info.amount),
"Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes),
"Funding tiers": strconv.Itoa(info.tiers),
@ -228,7 +228,7 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
return &faucetInfos{
node: &nodeInfos{
datadir: infos.volumes["/root/.faucet"],
portFull: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
port: infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
ethstats: infos.envvars["ETH_NAME"],
keyJSON: keyJSON,
keyPass: keyPass,

54
cmd/puppeth/module_node.go
View file

@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
RUN \
echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
ENTRYPOINT ["/bin/sh", "geth.sh"]
`
@ -56,15 +56,13 @@ services:
build: .
image: {{.Network}}/{{.Type}}
ports:
- "{{.FullPort}}:{{.FullPort}}"
- "{{.FullPort}}:{{.FullPort}}/udp"{{if .Light}}
- "{{.LightPort}}:{{.LightPort}}/udp"{{end}}
- "{{.Port}}:{{.Port}}"
- "{{.Port}}:{{.Port}}/udp"
volumes:
- {{.Datadir}}:/root/.ethereum{{if .Ethashdir}}
- {{.Ethashdir}}:/root/.ethash{{end}}
environment:
- FULL_PORT={{.FullPort}}/tcp
- LIGHT_PORT={{.LightPort}}/udp
- PORT={{.Port}}/tcp
- TOTAL_PEERS={{.TotalPeers}}
- LIGHT_PEERS={{.LightPeers}}
- STATS_NAME={{.Ethstats}}
@ -82,12 +80,11 @@ services:
// deployNode deploys a new Ethereum node container to a remote machine via SSH,
// docker and docker-compose. If an instance with the specified network name
// already exists there, it will be overwritten!
func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos, nocache bool) ([]byte, error) {
func deployNode(client *sshClient, network string, bootnodes []string, config *nodeInfos, nocache bool) ([]byte, error) {
kind := "sealnode"
if config.keyJSON == "" && config.etherbase == "" {
kind = "bootnode"
bootv4 = make([]string, 0)
bootv5 = make([]string, 0)
bootnodes = make([]string, 0)
}
// Generate the content to upload to the server
workdir := fmt.Sprintf("%d", rand.Int63())
@ -100,11 +97,10 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
dockerfile := new(bytes.Buffer)
template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
"NetworkID": config.network,
"Port": config.portFull,
"Port": config.port,
"Peers": config.peersTotal,
"LightFlag": lightFlag,
"BootV4": strings.Join(bootv4, ","),
"BootV5": strings.Join(bootv5, ","),
"Bootnodes": strings.Join(bootnodes, ","),
"Ethstats": config.ethstats,
"Etherbase": config.etherbase,
"GasTarget": uint64(1000000 * config.gasTarget),
@ -119,10 +115,9 @@ func deployNode(client *sshClient, network string, bootv4, bootv5 []string, conf
"Datadir": config.datadir,
"Ethashdir": config.ethashdir,
"Network": network,
"FullPort": config.portFull,
"Port": config.port,
"TotalPeers": config.peersTotal,
"Light": config.peersLight > 0,
"LightPort": config.portFull + 1,
"LightPeers": config.peersLight,
"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
"Etherbase": config.etherbase,
@ -157,10 +152,8 @@ type nodeInfos struct {
datadir string
ethashdir string
ethstats string
portFull int
portLight int
enodeFull string
enodeLight string
port int
enode string
peersTotal int
peersLight int
etherbase string
@ -174,15 +167,11 @@ type nodeInfos struct {
// most - but not all - fields for reporting to the user.
func (info *nodeInfos) Report() map[string]string {
report := map[string]string{
"Data directory": info.datadir,
"Listener port (full nodes)": strconv.Itoa(info.portFull),
"Peer count (all total)": strconv.Itoa(info.peersTotal),
"Peer count (light nodes)": strconv.Itoa(info.peersLight),
"Ethstats username": info.ethstats,
}
if info.peersLight > 0 {
// Light server enabled
report["Listener port (light nodes)"] = strconv.Itoa(info.portLight)
"Data directory": info.datadir,
"Listener port": strconv.Itoa(info.port),
"Peer count (all total)": strconv.Itoa(info.peersTotal),
"Peer count (light nodes)": strconv.Itoa(info.peersLight),
"Ethstats username": info.ethstats,
}
if info.gasTarget > 0 {
// Miner or signer node
@ -250,7 +239,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
keyPass = string(bytes.TrimSpace(out))
}
// Run a sanity check to see if the devp2p is reachable
port := infos.portmap[infos.envvars["FULL_PORT"]]
port := infos.portmap[infos.envvars["PORT"]]
if err = checkPort(client.server, port); err != nil {
log.Warn(fmt.Sprintf("%s devp2p port seems unreachable", strings.Title(kind)), "server", client.server, "port", port, "err", err)
}
@ -259,8 +248,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
genesis: genesis,
datadir: infos.volumes["/root/.ethereum"],
ethashdir: infos.volumes["/root/.ethash"],
portFull: infos.portmap[infos.envvars["FULL_PORT"]],
portLight: infos.portmap[infos.envvars["LIGHT_PORT"]],
port: port,
peersTotal: totalPeers,
peersLight: lightPeers,
ethstats: infos.envvars["STATS_NAME"],
@ -270,9 +258,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
gasTarget: gasTarget,
gasPrice: gasPrice,
}
stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull)
if stats.portLight != 0 {
stats.enodeLight = fmt.Sprintf("enode://%s@%s:%d?discport=%d", id, client.address, stats.portFull, stats.portLight)
}
stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
return stats, nil
}

3
cmd/puppeth/wizard.go
View file

@ -40,8 +40,7 @@ import (
// between sessions.
type config struct {
path string // File containing the configuration values
bootFull []string // Bootnodes to always connect to by full nodes
bootLight []string // Bootnodes to always connect to by light nodes
bootnodes []string // Bootnodes to always connect to by all nodes
ethstats string // Ethstats settings to cache for node deploys
Genesis *core.Genesis `json:"genesis,omitempty"` // Genesis block to cache for node deploys

2
cmd/puppeth/wizard_explorer.go
View file

@ -55,7 +55,7 @@ func (w *wizard) deployExplorer() {
}
existed := err == nil
chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootFull)
chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes)
if err != nil {
log.Error("Failed to create chain spec for explorer", "err", err)
return

8
cmd/puppeth/wizard_faucet.go
View file

@ -38,7 +38,7 @@ func (w *wizard) deployFaucet() {
infos, err := checkFaucet(client, w.network)
if err != nil {
infos = &faucetInfos{
node: &nodeInfos{portFull: 30303, peersTotal: 25},
node: &nodeInfos{port: 30303, peersTotal: 25},
port: 80,
host: client.server,
amount: 1,
@ -113,8 +113,8 @@ func (w *wizard) deployFaucet() {
}
// Figure out which port to listen on
fmt.Println()
fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.portFull)
infos.node.portFull = w.readDefaultInt(infos.node.portFull)
fmt.Printf("Which TCP/UDP port should the light client listen on? (default = %d)\n", infos.node.port)
infos.node.port = w.readDefaultInt(infos.node.port)
// Set a proper name to report on the stats page
fmt.Println()
@ -168,7 +168,7 @@ func (w *wizard) deployFaucet() {
fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n")
nocache = w.readDefaultString("n") != "n"
}
if out, err := deployFaucet(client, w.network, w.conf.bootLight, infos, nocache); err != nil {
if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
log.Error("Failed to deploy faucet container", "err", err)
if len(out) > 0 {
fmt.Printf("%s\n", out)

14
cmd/puppeth/wizard_netstats.go
View file

@ -37,8 +37,7 @@ func (w *wizard) networkStats() {
}
// Clear out some previous configs to refill from current scan
w.conf.ethstats = ""
w.conf.bootFull = w.conf.bootFull[:0]
w.conf.bootLight = w.conf.bootLight[:0]
w.conf.bootnodes = w.conf.bootnodes[:0]
// Iterate over all the specified hosts and check their status
var pend sync.WaitGroup
@ -76,8 +75,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
var (
genesis string
ethstats string
bootFull []string
bootLight []string
bootnodes []string
)
// Ensure a valid SSH connection to the remote server
logger := log.New("server", server)
@ -123,10 +121,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
stat.services["bootnode"] = infos.Report()
genesis = string(infos.genesis)
bootFull = append(bootFull, infos.enodeFull)
if infos.enodeLight != "" {
bootLight = append(bootLight, infos.enodeLight)
}
bootnodes = append(bootnodes, infos.enode)
}
logger.Debug("Checking for sealnode availability")
if infos, err := checkNode(client, w.network, false); err != nil {
@ -184,8 +179,7 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s
if ethstats != "" {
w.conf.ethstats = ethstats
}
w.conf.bootFull = append(w.conf.bootFull, bootFull...)
w.conf.bootLight = append(w.conf.bootLight, bootLight...)
w.conf.bootnodes = append(w.conf.bootnodes, bootnodes...)
return stat
}

10
cmd/puppeth/wizard_node.go
View file

@ -48,9 +48,9 @@ func (w *wizard) deployNode(boot bool) {
infos, err := checkNode(client, w.network, boot)
if err != nil {
if boot {
infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256}
infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256}
} else {
infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
}
}
existed := err == nil
@ -79,8 +79,8 @@ func (w *wizard) deployNode(boot bool) {
}
// Figure out which port to listen on
fmt.Println()
fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.portFull)
infos.portFull = w.readDefaultInt(infos.portFull)
fmt.Printf("Which TCP/UDP port to listen on? (default = %d)\n", infos.port)
infos.port = w.readDefaultInt(infos.port)
// Figure out how many peers to allow (different based on node type)
fmt.Println()
@ -163,7 +163,7 @@ func (w *wizard) deployNode(boot bool) {
fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n")
nocache = w.readDefaultString("n") != "n"
}
if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos, nocache); err != nil {
if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
log.Error("Failed to deploy Ethereum node container", "err", err)
if len(out) > 0 {
fmt.Printf("%s\n", out)

2
cmd/puppeth/wizard_wallet.go
View file

@ -98,7 +98,7 @@ func (w *wizard) deployWallet() {
fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n")
nocache = w.readDefaultString("n") != "n"
}
if out, err := deployWallet(client, w.network, w.conf.bootFull, infos, nocache); err != nil {
if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
log.Error("Failed to deploy wallet container", "err", err)
if len(out) > 0 {
fmt.Printf("%s\n", out)

2
cmd/swarm/run_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify

2
cmd/swarm/upload_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify

31
cmd/utils/cmd.go
View file

@ -25,6 +25,7 @@ import (
"os/signal"
"runtime"
"strings"
"syscall"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
@ -64,7 +65,7 @@ func StartNode(stack *node.Node) {
}
go func() {
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, os.Interrupt)
signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(sigc)
<-sigc
log.Info("Got interrupt, shutting down...")
@ -85,7 +86,7 @@ func ImportChain(chain *core.BlockChain, fn string) error {
// If a signal is received, the import will stop at the next batch.
interrupt := make(chan os.Signal, 1)
stop := make(chan struct{})
signal.Notify(interrupt, os.Interrupt)
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
defer signal.Stop(interrupt)
defer close(interrupt)
go func() {
@ -116,7 +117,6 @@ func ImportChain(chain *core.BlockChain, fn string) error {
return err
}
}
stream := rlp.NewStream(reader, 0)
// Run actual the import.
@ -150,25 +150,34 @@ func ImportChain(chain *core.BlockChain, fn string) error {
if checkInterrupt() {
return fmt.Errorf("interrupted")
}
if hasAllBlocks(chain, blocks[:i]) {
missing := missingBlocks(chain, blocks[:i])
if len(missing) == 0 {
log.Info("Skipping batch as all blocks present", "batch", batch, "first", blocks[0].Hash(), "last", blocks[i-1].Hash())
continue
}
if _, err := chain.InsertChain(blocks[:i]); err != nil {
if _, err := chain.InsertChain(missing); err != nil {
return fmt.Errorf("invalid block %d: %v", n, err)
}
}
return nil
}
func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool {
for _, b := range bs {
if !chain.HasBlock(b.Hash(), b.NumberU64()) {
return false
func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block {
head := chain.CurrentBlock()
for i, block := range blocks {
// If we're behind the chain head, only check block, state is available at head
if head.NumberU64() > block.NumberU64() {
if !chain.HasBlock(block.Hash(), block.NumberU64()) {
return blocks[i:]
}
continue
}
// If we're above the chain head, state availability is a must
if !chain.HasBlockAndState(block.Hash(), block.NumberU64()) {
return blocks[i:]
}
}
return true
return nil
}
func ExportChain(blockchain *core.BlockChain, fn string) error {

90
cmd/utils/flags.go
View file

@ -170,7 +170,11 @@ var (
Usage: `Blockchain sync mode ("fast", "full", or "light")`,
Value: &defaultSyncMode,
}
GCModeFlag = cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
}
LightServFlag = cli.IntFlag{
Name: "lightserv",
Usage: "Maximum percentage of time allowed for serving LES requests (0-90)",
@ -179,7 +183,7 @@ var (
LightPeersFlag = cli.IntFlag{
Name: "lightpeers",
Usage: "Maximum number of LES client peers",
Value: 20,
Value: eth.DefaultConfig.LightPeers,
}
LightKDFFlag = cli.BoolFlag{
Name: "lightkdf",
@ -293,8 +297,18 @@ var (
// Performance tuning settings
CacheFlag = cli.IntFlag{
Name: "cache",
Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)",
Value: 128,
Usage: "Megabytes of memory allocated to internal caching",
Value: 1024,
}
CacheDatabaseFlag = cli.IntFlag{
Name: "cache.database",
Usage: "Percentage of cache memory allowance to use for database io",
Value: 75,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
Usage: "Percentage of cache memory allowance to use for trie pruning",
Value: 25,
}
TrieCacheGenFlag = cli.IntFlag{
Name: "trie-cache-gens",
@ -383,6 +397,11 @@ var (
Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
Value: "",
}
RPCVirtualHostsFlag = cli.StringFlag{
Name: "rpcvhosts",
Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.",
Value: "localhost",
}
RPCApiFlag = cli.StringFlag{
Name: "rpcapi",
Usage: "API's offered over the HTTP-RPC interface",
@ -612,7 +631,7 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
}
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyV5Bootnodes
urls = params.RinkebyBootnodes
case cfg.BootstrapNodesV5 != nil:
return // already set, don't apply defaults.
}
@ -676,6 +695,8 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) {
if ctx.GlobalIsSet(RPCApiFlag.Name) {
cfg.HTTPModules = splitAndTrim(ctx.GlobalString(RPCApiFlag.Name))
}
cfg.HTTPVirtualHosts = splitAndTrim(ctx.GlobalString(RPCVirtualHostsFlag.Name))
}
// setWS creates the WebSocket RPC listener interface string from the set
@ -714,13 +735,15 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
// makeDatabaseHandles raises out the number of allowed file handles per process
// for Geth and returns half of the allowance to assign to the database.
func makeDatabaseHandles() int {
if err := fdlimit.Raise(2048); err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
limit, err := fdlimit.Current()
if err != nil {
Fatalf("Failed to retrieve file descriptor allowance: %v", err)
}
if limit < 2048 {
if err := fdlimit.Raise(2048); err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
}
if limit > 2048 { // cap database file descriptors even if more is available
limit = 2048
}
@ -789,20 +812,40 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
setBootstrapNodes(ctx, cfg)
setBootstrapNodesV5(ctx, cfg)
lightClient := ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalString(SyncModeFlag.Name) == "light"
lightServer := ctx.GlobalInt(LightServFlag.Name) != 0
lightPeers := ctx.GlobalInt(LightPeersFlag.Name)
if ctx.GlobalIsSet(MaxPeersFlag.Name) {
cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
} else {
if lightServer {
cfg.MaxPeers += lightPeers
}
if lightClient && ctx.GlobalIsSet(LightPeersFlag.Name) && cfg.MaxPeers < lightPeers {
cfg.MaxPeers = lightPeers
}
}
if !(lightClient || lightServer) {
lightPeers = 0
}
ethPeers := cfg.MaxPeers - lightPeers
if lightClient {
ethPeers = 0
}
log.Info("Maximum peer count", "ETH", ethPeers, "LES", lightPeers, "total", cfg.MaxPeers)
if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) {
cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name)
}
if ctx.GlobalIsSet(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name) {
if ctx.GlobalIsSet(NoDiscoverFlag.Name) || lightClient {
cfg.NoDiscovery = true
}
// if we're running a light client or server, force enable the v5 peer discovery
// unless it is explicitly disabled with --nodiscover note that explicitly specifying
// --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name)
forceV5Discovery := (lightClient || lightServer) && !ctx.GlobalBool(NoDiscoverFlag.Name)
if ctx.GlobalIsSet(DiscoveryV5Flag.Name) {
cfg.DiscoveryV5 = ctx.GlobalBool(DiscoveryV5Flag.Name)
} else if forceV5Discovery {
@ -999,11 +1042,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
}
if ctx.GlobalIsSet(CacheFlag.Name) {
cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name)
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
}
cfg.DatabaseHandles = makeDatabaseHandles()
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
}
@ -1135,7 +1186,7 @@ func SetupNetwork(ctx *cli.Context) {
// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
var (
cache = ctx.GlobalInt(CacheFlag.Name)
cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
handles = makeDatabaseHandles()
)
name := "chaindata"
@ -1187,8 +1238,19 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
})
}
}
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cache := &core.CacheConfig{
Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieNodeLimit: eth.DefaultConfig.TrieCache,
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)
if err != nil {
Fatalf("Can't create BlockChain: %v", err)
}

65
cmd/wnode/main.go
View file

@ -43,7 +43,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/whisper/mailserver"
whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
"golang.org/x/crypto/pbkdf2"
)
@ -61,15 +61,17 @@ var (
// encryption
var (
symKey []byte
pub *ecdsa.PublicKey
asymKey *ecdsa.PrivateKey
nodeid *ecdsa.PrivateKey
topic whisper.TopicType
asymKeyID string
filterID string
symPass string
msPassword string
symKey []byte
pub *ecdsa.PublicKey
asymKey *ecdsa.PrivateKey
nodeid *ecdsa.PrivateKey
topic whisper.TopicType
asymKeyID string
asymFilterID string
symFilterID string
symPass string
msPassword string
)
// cmd arguments
@ -263,7 +265,7 @@ func initialize() {
Config: p2p.Config{
PrivateKey: nodeid,
MaxPeers: maxPeers,
Name: common.MakeName("wnode", "5.0"),
Name: common.MakeName("wnode", "6.0"),
Protocols: shh.Protocols(),
ListenAddr: *argIP,
NAT: nat.Any(),
@ -363,13 +365,22 @@ func configureNode() {
}
}
filter := whisper.Filter{
symFilter := whisper.Filter{
KeySym: symKey,
Topics: [][]byte{topic[:]},
AllowP2P: p2pAccept,
}
symFilterID, err = shh.Subscribe(&symFilter)
if err != nil {
utils.Fatalf("Failed to install filter: %s", err)
}
asymFilter := whisper.Filter{
KeyAsym: asymKey,
Topics: [][]byte{topic[:]},
AllowP2P: p2pAccept,
}
filterID, err = shh.Subscribe(&filter)
asymFilterID, err = shh.Subscribe(&asymFilter)
if err != nil {
utils.Fatalf("Failed to install filter: %s", err)
}
@ -522,9 +533,14 @@ func sendMsg(payload []byte) common.Hash {
}
func messageLoop() {
f := shh.GetFilter(filterID)
if f == nil {
utils.Fatalf("filter is not installed")
sf := shh.GetFilter(symFilterID)
if sf == nil {
utils.Fatalf("symmetric filter is not installed")
}
af := shh.GetFilter(asymFilterID)
if af == nil {
utils.Fatalf("asymmetric filter is not installed")
}
ticker := time.NewTicker(time.Millisecond * 50)
@ -532,7 +548,16 @@ func messageLoop() {
for {
select {
case <-ticker.C:
messages := f.Retrieve()
messages := sf.Retrieve()
for _, msg := range messages {
if *fileExMode || len(msg.Payload) > 2048 {
writeMessageToFile(*argSaveDir, msg)
} else {
printMessageInfo(msg)
}
}
messages = af.Retrieve()
for _, msg := range messages {
if *fileExMode || len(msg.Payload) > 2048 {
writeMessageToFile(*argSaveDir, msg)
@ -601,7 +626,7 @@ func requestExpiredMessagesLoop() {
if err != nil {
utils.Fatalf("Failed to save symmetric key for mail request: %s", err)
}
peerID = extractIdFromEnode(*argEnode)
peerID = extractIDFromEnode(*argEnode)
shh.AllowP2PMessagesFromPeer(peerID)
for {
@ -631,7 +656,7 @@ func requestExpiredMessagesLoop() {
params.PoW = *argServerPoW
params.Payload = data
params.KeySym = key
params.Src = nodeid
params.Src = asymKey
params.WorkTime = 5
msg, err := whisper.NewSentMessage(&params)
@ -652,7 +677,7 @@ func requestExpiredMessagesLoop() {
}
}
func extractIdFromEnode(s string) []byte {
func extractIDFromEnode(s string) []byte {
n, err := discover.ParseNode(s)
if err != nil {
utils.Fatalf("Failed to parse enode: %s", err)

14
common/fdlimit/fdlimit_freebsd.go
View file

@ -1,18 +1,18 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build freebsd

14
common/fdlimit/fdlimit_test.go
View file

@ -1,18 +1,18 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fdlimit

14
common/fdlimit/fdlimit_unix.go
View file

@ -1,18 +1,18 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin netbsd openbsd solaris

16
common/fdlimit/fdlimit_windows.go
View file

@ -1,18 +1,18 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fdlimit

27
common/size.go
View file

@ -20,18 +20,29 @@ import (
"fmt"
)
// StorageSize is a wrapper around a float value that supports user friendly
// formatting.
type StorageSize float64
func (self StorageSize) String() string {
if self > 1000000 {
return fmt.Sprintf("%.2f mB", self/1000000)
} else if self > 1000 {
return fmt.Sprintf("%.2f kB", self/1000)
// String implements the stringer interface.
func (s StorageSize) String() string {
if s > 1000000 {
return fmt.Sprintf("%.2f mB", s/1000000)
} else if s > 1000 {
return fmt.Sprintf("%.2f kB", s/1000)
} else {
return fmt.Sprintf("%.2f B", self)
return fmt.Sprintf("%.2f B", s)
}
}
func (self StorageSize) Int64() int64 {
return int64(self)
// TerminalString implements log.TerminalStringer, formatting a string for console
// output during logging.
func (s StorageSize) TerminalString() string {
if s > 1000000 {
return fmt.Sprintf("%.2fmB", s/1000000)
} else if s > 1000 {
return fmt.Sprintf("%.2fkB", s/1000)
} else {
return fmt.Sprintf("%.2fB", s)
}
}

4
consensus/errors.go
View file

@ -23,6 +23,10 @@ var (
// that is unknown.
ErrUnknownAncestor = errors.New("unknown ancestor")
// ErrPrunedAncestor is returned when validating a block requires an ancestor
// that is known, but the state of which is not available.
ErrPrunedAncestor = errors.New("pruned ancestor")
// ErrFutureBlock is returned when a block's timestamp is in the future according
// to the current node.
ErrFutureBlock = errors.New("block in the future")

3
console/console.go
View file

@ -26,6 +26,7 @@ import (
"regexp"
"sort"
"strings"
"syscall"
"github.com/ethereum/go-ethereum/internal/jsre"
"github.com/ethereum/go-ethereum/internal/web3ext"
@ -332,7 +333,7 @@ func (c *Console) Interactive() {
}()
// Monitor Ctrl-C too in case the input is empty and we need to bail
abort := make(chan os.Signal, 1)
signal.Notify(abort, os.Interrupt)
signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
// Start sending prompts to the user and reading back inputs
for {

2
containers/docker/master-alpine/Dockerfile
View file

@ -2,7 +2,7 @@ FROM alpine:3.7
RUN \
apk add --update go git make gcc musl-dev linux-headers ca-certificates && \
git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
(cd go-ethereum && make geth) && \
cp go-ethereum/build/bin/geth /geth && \
apk del go git make gcc musl-dev linux-headers && \

2
containers/docker/master-ubuntu/Dockerfile
View file

@ -5,7 +5,7 @@ ENV PATH=/usr/lib/go-1.9/bin:$PATH
RUN \
apt-get update && apt-get upgrade -q -y && \
apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \
git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \
git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \
(cd go-ethereum && make geth) && \
cp go-ethereum/build/bin/geth /geth && \
apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \

4
core/bench_test.go
View file

@ -173,7 +173,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
// Time the insertion of the new chain.
// State and blocks are stored in the same DB.
chainman, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer chainman.Stop()
b.ReportAllocs()
b.ResetTimer()
@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
}
chain, err := NewBlockChain(db, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
if err != nil {
b.Fatalf("error creating chain: %v", err)
}

9
core/block_validator.go
View file

@ -50,11 +50,14 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin
// validated at this point.
func (v *BlockValidator) ValidateBody(block *types.Block) error {
// Check whether the block's known, and if not, that it's linkable
if v.bc.HasBlockAndState(block.Hash()) {
if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
return ErrKnownBlock
}
if !v.bc.HasBlockAndState(block.ParentHash()) {
return consensus.ErrUnknownAncestor
if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
return consensus.ErrUnknownAncestor
}
return consensus.ErrPrunedAncestor
}
// Header validity is known at this point, check the uncles and transactions
header := block.Header()

8
core/block_validator_test.go
View file

@ -42,7 +42,7 @@ func TestHeaderVerification(t *testing.T) {
headers[i] = block.Header()
}
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
defer chain.Stop()
for i := 0; i < len(blocks); i++ {
@ -106,11 +106,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) {
var results <-chan error
if valid {
chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
_, results = chain.engine.VerifyHeaders(chain, headers, seals)
chain.Stop()
} else {
chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{})
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{})
_, results = chain.engine.VerifyHeaders(chain, headers, seals)
chain.Stop()
}
@ -173,7 +173,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) {
defer runtime.GOMAXPROCS(old)
// Start the verifications and immediately abort
chain, _ := NewBlockChain(testdb, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{})
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{})
defer chain.Stop()
abort, results := chain.engine.VerifyHeaders(chain, headers, seals)

351
core/blockchain.go
View file

@ -42,6 +42,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/hashicorp/golang-lru"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)
var (
@ -56,11 +57,20 @@ const (
maxFutureBlocks = 256
maxTimeFutureBlocks = 30
badBlockLimit = 10
triesInMemory = 128
// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
BlockChainVersion = 3
)
// CacheConfig contains the configuration values for the trie caching/pruning
// that's resident in a blockchain.
type CacheConfig struct {
Disabled bool // Whether to disable trie write caching (archive node)
TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
}
// BlockChain represents the canonical chain given a database with a genesis
// block. The Blockchain manages chain imports, reverts, chain reorganisations.
//
@ -76,10 +86,14 @@ const (
// included in the canonical one where as GetBlockByNumber always represents the
// canonical chain.
type BlockChain struct {
config *params.ChainConfig // chain & network configuration
chainConfig *params.ChainConfig // Chain & network configuration
cacheConfig *CacheConfig // Cache configuration for pruning
db ethdb.Database // Low level persistent database to store final content in
triegc *prque.Prque // Priority queue mapping block numbers to tries to gc
gcproc time.Duration // Accumulates canonical block processing for trie dumping
hc *HeaderChain
chainDb ethdb.Database
rmLogsFeed event.Feed
chainFeed event.Feed
chainSideFeed event.Feed
@ -119,7 +133,13 @@ type BlockChain struct {
// NewBlockChain returns a fully initialised block chain using information
// available in the database. It initialises the default Ethereum Validator and
// Processor.
func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
if cacheConfig == nil {
cacheConfig = &CacheConfig{
TrieNodeLimit: 256 * 1024 * 1024,
TrieTimeLimit: 5 * time.Minute,
}
}
bodyCache, _ := lru.New(bodyCacheLimit)
bodyRLPCache, _ := lru.New(bodyCacheLimit)
blockCache, _ := lru.New(blockCacheLimit)
@ -127,9 +147,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
badBlocks, _ := lru.New(badBlockLimit)
bc := &BlockChain{
config: config,
chainDb: chainDb,
stateCache: state.NewDatabase(chainDb),
chainConfig: chainConfig,
cacheConfig: cacheConfig,
db: db,
triegc: prque.New(),
stateCache: state.NewDatabase(db),
quit: make(chan struct{}),
bodyCache: bodyCache,
bodyRLPCache: bodyRLPCache,
@ -139,11 +161,11 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
vmConfig: vmConfig,
badBlocks: badBlocks,
}
bc.SetValidator(NewBlockValidator(config, bc, engine))
bc.SetProcessor(NewStateProcessor(config, bc, engine))
bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
var err error
bc.hc, err = NewHeaderChain(chainDb, config, engine, bc.getProcInterrupt)
bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.getProcInterrupt)
if err != nil {
return nil, err
}
@ -180,7 +202,7 @@ func (bc *BlockChain) getProcInterrupt() bool {
// assumes that the chain manager mutex is held.
func (bc *BlockChain) loadLastState() error {
// Restore the last known head block
head := GetHeadBlockHash(bc.chainDb)
head := GetHeadBlockHash(bc.db)
if head == (common.Hash{}) {
// Corrupt or empty database, init from scratch
log.Warn("Empty database, resetting chain")
@ -196,15 +218,17 @@ func (bc *BlockChain) loadLastState() error {
// Make sure the state associated with the block is available
if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
// Dangling block without a state associated, init from scratch
log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
return bc.Reset()
log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
if err := bc.repair(&currentBlock); err != nil {
return err
}
}
// Everything seems to be fine, set as the head block
bc.currentBlock = currentBlock
// Restore the last known head header
currentHeader := bc.currentBlock.Header()
if head := GetHeadHeaderHash(bc.chainDb); head != (common.Hash{}) {
if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) {
if header := bc.GetHeaderByHash(head); header != nil {
currentHeader = header
}
@ -213,7 +237,7 @@ func (bc *BlockChain) loadLastState() error {
// Restore the last known head fast block
bc.currentFastBlock = bc.currentBlock
if head := GetHeadFastBlockHash(bc.chainDb); head != (common.Hash{}) {
if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) {
if block := bc.GetBlockByHash(head); block != nil {
bc.currentFastBlock = block
}
@ -243,7 +267,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
// Rewind the header chain, deleting all block bodies until then
delFn := func(hash common.Hash, num uint64) {
DeleteBody(bc.chainDb, hash, num)
DeleteBody(bc.db, hash, num)
}
bc.hc.SetHead(head, delFn)
currentHeader := bc.hc.CurrentHeader()
@ -275,10 +299,10 @@ func (bc *BlockChain) SetHead(head uint64) error {
if bc.currentFastBlock == nil {
bc.currentFastBlock = bc.genesisBlock
}
if err := WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()); err != nil {
if err := WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()); err != nil {
log.Crit("Failed to reset head full block", "err", err)
}
if err := WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()); err != nil {
if err := WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()); err != nil {
log.Crit("Failed to reset head fast block", "err", err)
}
return bc.loadLastState()
@ -292,7 +316,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x…]", hash[:4])
}
if _, err := trie.NewSecure(block.Root(), bc.chainDb, 0); err != nil {
if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB(), 0); err != nil {
return err
}
// If all checks out, manually set the head block
@ -312,14 +336,6 @@ func (bc *BlockChain) GasLimit() uint64 {
return bc.currentBlock.GasLimit()
}
// LastBlockHash return the hash of the HEAD block.
func (bc *BlockChain) LastBlockHash() common.Hash {
bc.mu.RLock()
defer bc.mu.RUnlock()
return bc.currentBlock.Hash()
}
// CurrentBlock retrieves the current head block of the canonical chain. The
// block is retrieved from the blockchain's internal cache.
func (bc *BlockChain) CurrentBlock() *types.Block {
@ -338,15 +354,6 @@ func (bc *BlockChain) CurrentFastBlock() *types.Block {
return bc.currentFastBlock
}
// Status returns status information about the current chain such as the HEAD Td,
// the HEAD hash and the hash of the genesis block.
func (bc *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) {
bc.mu.RLock()
defer bc.mu.RUnlock()
return bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()), bc.currentBlock.Hash(), bc.genesisBlock.Hash()
}
// SetProcessor sets the processor required for making state modifications.
func (bc *BlockChain) SetProcessor(processor Processor) {
bc.procmu.Lock()
@ -404,7 +411,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil {
log.Crit("Failed to write genesis block TD", "err", err)
}
if err := WriteBlock(bc.chainDb, genesis); err != nil {
if err := WriteBlock(bc.db, genesis); err != nil {
log.Crit("Failed to write genesis block", "err", err)
}
bc.genesisBlock = genesis
@ -417,6 +424,24 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
return nil
}
// repair tries to repair the current blockchain by rolling back the current block
// until one with associated state is found. This is needed to fix incomplete db
// writes caused either by crashes/power outages, or simply non-committed tries.
//
// This method only rolls back the current block. The current header and current
// fast block are left intact.
func (bc *BlockChain) repair(head **types.Block) error {
for {
// Abort if we've rewound to a head block that does have associated state
if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash())
return nil
}
// Otherwise rewind one block and recheck state availability there
(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
}
}
// Export writes the active chain to the given writer.
func (bc *BlockChain) Export(w io.Writer) error {
return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64())
@ -454,13 +479,13 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
// Note, this function assumes that the `mu` mutex is held!
func (bc *BlockChain) insert(block *types.Block) {
// If the block is on a side chain or an unknown one, force other heads onto it too
updateHeads := GetCanonicalHash(bc.chainDb, block.NumberU64()) != block.Hash()
updateHeads := GetCanonicalHash(bc.db, block.NumberU64()) != block.Hash()
// Add the block to the canonical chain number scheme and mark as the head
if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil {
if err := WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()); err != nil {
log.Crit("Failed to insert block number", "err", err)
}
if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil {
if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil {
log.Crit("Failed to insert head block hash", "err", err)
}
bc.currentBlock = block
@ -469,7 +494,7 @@ func (bc *BlockChain) insert(block *types.Block) {
if updateHeads {
bc.hc.SetCurrentHeader(block.Header())
if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil {
if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil {
log.Crit("Failed to insert head fast block hash", "err", err)
}
bc.currentFastBlock = block
@ -489,7 +514,7 @@ func (bc *BlockChain) GetBody(hash common.Hash) *types.Body {
body := cached.(*types.Body)
return body
}
body := GetBody(bc.chainDb, hash, bc.hc.GetBlockNumber(hash))
body := GetBody(bc.db, hash, bc.hc.GetBlockNumber(hash))
if body == nil {
return nil
}
@ -505,7 +530,7 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue {
if cached, ok := bc.bodyRLPCache.Get(hash); ok {
return cached.(rlp.RawValue)
}
body := GetBodyRLP(bc.chainDb, hash, bc.hc.GetBlockNumber(hash))
body := GetBodyRLP(bc.db, hash, bc.hc.GetBlockNumber(hash))
if len(body) == 0 {
return nil
}
@ -519,21 +544,25 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
if bc.blockCache.Contains(hash) {
return true
}
ok, _ := bc.chainDb.Has(blockBodyKey(hash, number))
ok, _ := bc.db.Has(blockBodyKey(hash, number))
return ok
}
// HasState checks if state trie is fully present in the database or not.
func (bc *BlockChain) HasState(hash common.Hash) bool {
_, err := bc.stateCache.OpenTrie(hash)
return err == nil
}
// HasBlockAndState checks if a block and associated state trie is fully present
// in the database or not, caching it if present.
func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool {
func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
// Check first that the block itself is known
block := bc.GetBlockByHash(hash)
block := bc.GetBlock(hash, number)
if block == nil {
return false
}
// Ensure the associated state is also present
_, err := bc.stateCache.OpenTrie(block.Root())
return err == nil
return bc.HasState(block.Root())
}
// GetBlock retrieves a block from the database by hash and number,
@ -543,7 +572,7 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block {
if block, ok := bc.blockCache.Get(hash); ok {
return block.(*types.Block)
}
block := GetBlock(bc.chainDb, hash, number)
block := GetBlock(bc.db, hash, number)
if block == nil {
return nil
}
@ -560,13 +589,18 @@ func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block {
// GetBlockByNumber retrieves a block from the database by number, caching it
// (associated with its hash) if found.
func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block {
hash := GetCanonicalHash(bc.chainDb, number)
hash := GetCanonicalHash(bc.db, number)
if hash == (common.Hash{}) {
return nil
}
return bc.GetBlock(hash, number)
}
// GetReceiptsByHash retrieves the receipts for all transactions in a given block.
func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
return GetBlockReceipts(bc.db, hash, GetBlockNumber(bc.db, hash))
}
// GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.
// [deprecated by eth/62]
func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) {
@ -594,6 +628,12 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.
return uncles
}
// TrieNode retrieves a blob of data associated with a trie node (or code hash)
// either from ephemeral in-memory cache, or from persistent storage.
func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
return bc.stateCache.TrieDB().Node(hash)
}
// Stop stops the blockchain service. If any imports are currently in progress
// it will abort them using the procInterrupt.
func (bc *BlockChain) Stop() {
@ -606,6 +646,33 @@ func (bc *BlockChain) Stop() {
atomic.StoreInt32(&bc.procInterrupt, 1)
bc.wg.Wait()
// Ensure the state of a recent block is also stored to disk before exiting.
// It is fine if this state does not exist (fast start/stop cycle), but it is
// advisable to leave an N block gap from the head so 1) a restart loads up
// the last N blocks as sync assistance to remote nodes; 2) a restart during
// a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from
// missing states are constantly tested.
//
// This may be tuned a bit on mainnet if its too annoying to reprocess the last
// N blocks.
if !bc.cacheConfig.Disabled {
triedb := bc.stateCache.TrieDB()
if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory {
recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1)
log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
if err := triedb.Commit(recent.Root(), true); err != nil {
log.Error("Failed to commit recent state trie", "err", err)
}
}
for !bc.triegc.Empty() {
triedb.Dereference(bc.triegc.PopItem().(common.Hash), common.Hash{})
}
if size := triedb.Size(); size != 0 {
log.Error("Dangling trie nodes after full cleanup")
}
}
log.Info("Blockchain manager stopped")
}
@ -650,11 +717,11 @@ func (bc *BlockChain) Rollback(chain []common.Hash) {
}
if bc.currentFastBlock.Hash() == hash {
bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1)
WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash())
WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash())
}
if bc.currentBlock.Hash() == hash {
bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1)
WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash())
WriteHeadBlockHash(bc.db, bc.currentBlock.Hash())
}
}
}
@ -713,7 +780,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
stats = struct{ processed, ignored int32 }{}
start = time.Now()
bytes = 0
batch = bc.chainDb.NewBatch()
batch = bc.db.NewBatch()
)
for i, block := range blockChain {
receipts := receiptChain[i]
@ -731,7 +798,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
continue
}
// Compute all the non-consensus fields of the receipts
SetReceiptsData(bc.config, block, receipts)
SetReceiptsData(bc.chainConfig, block, receipts)
// Write all the data out into the database
if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil {
return i, fmt.Errorf("failed to write block body: %v", err)
@ -749,7 +816,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
return 0, err
}
bytes += batch.ValueSize()
batch = bc.chainDb.NewBatch()
batch.Reset()
}
}
if batch.ValueSize() > 0 {
@ -764,7 +831,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
head := blockChain[len(blockChain)-1]
if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case
if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 {
if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil {
if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil {
log.Crit("Failed to update head fast block hash", "err", err)
}
bc.currentFastBlock = head
@ -775,15 +842,33 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
log.Info("Imported new block receipts",
"count", stats.processed,
"elapsed", common.PrettyDuration(time.Since(start)),
"bytes", bytes,
"number", head.Number(),
"hash", head.Hash(),
"size", common.StorageSize(bytes),
"ignored", stats.ignored)
return 0, nil
}
// WriteBlock writes the block to the chain.
func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) {
var lastWrite uint64
// WriteBlockWithoutState writes only the block and its metadata to the database,
// but does not write any state. This is used to construct competing side forks
// up to the point where they exceed the canonical total difficulty.
func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (err error) {
bc.wg.Add(1)
defer bc.wg.Done()
if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil {
return err
}
if err := WriteBlock(bc.db, block); err != nil {
return err
}
return nil
}
// WriteBlockWithState writes the block and all associated state to the database.
func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) {
bc.wg.Add(1)
defer bc.wg.Done()
@ -804,17 +889,69 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
return NonStatTy, err
}
// Write other block data using a batch.
batch := bc.chainDb.NewBatch()
batch := bc.db.NewBatch()
if err := WriteBlock(batch, block); err != nil {
return NonStatTy, err
}
if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil {
root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number()))
if err != nil {
return NonStatTy, err
}
triedb := bc.stateCache.TrieDB()
// If we're running an archive node, always flush
if bc.cacheConfig.Disabled {
if err := triedb.Commit(root, false); err != nil {
return NonStatTy, err
}
} else {
// Full but not archive node, do proper garbage collection
triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
bc.triegc.Push(root, -float32(block.NumberU64()))
if current := block.NumberU64(); current > triesInMemory {
// Find the next state trie we need to commit
header := bc.GetHeaderByNumber(current - triesInMemory)
chosen := header.Number.Uint64()
// Only write to disk if we exceeded our memory allowance *and* also have at
// least a given number of tries gapped.
var (
size = triedb.Size()
limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
)
if size > limit || bc.gcproc > bc.cacheConfig.TrieTimeLimit {
// If we're exceeding limits but haven't reached a large enough memory gap,
// warn the user that the system is becoming unstable.
if chosen < lastWrite+triesInMemory {
switch {
case size >= 2*limit:
log.Warn("State memory usage too high, committing", "size", size, "limit", limit, "optimum", float64(chosen-lastWrite)/triesInMemory)
case bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit:
log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory)
}
}
// If optimum or critical limits reached, write to disk
if chosen >= lastWrite+triesInMemory || size >= 2*limit || bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
triedb.Commit(header.Root, true)
lastWrite = chosen
bc.gcproc = 0
}
}
// Garbage collect anything below our required write retention
for !bc.triegc.Empty() {
root, number := bc.triegc.Pop()
if uint64(-number) > chosen {
bc.triegc.Push(root, number)
break
}
triedb.Dereference(root.(common.Hash), common.Hash{})
}
}
}
if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
return NonStatTy, err
}
// If the total difficulty is higher than our known, add it to the canonical chain
// Second clause in the if statement reduces the vulnerability to selfish mining.
// Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf
@ -835,7 +972,7 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
return NonStatTy, err
}
// Write hash preimages
if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil {
if err := WritePreimages(bc.db, block.NumberU64(), state.Preimages()); err != nil {
return NonStatTy, err
}
status = CanonStatTy
@ -927,31 +1064,64 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
if err == nil {
err = bc.Validator().ValidateBody(block)
}
if err != nil {
if err == ErrKnownBlock {
switch {
case err == ErrKnownBlock:
// Block and state both already known. However if the current block is below
// this number we did a rollback and we should reimport it nonetheless.
if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
stats.ignored++
continue
}
if err == consensus.ErrFutureBlock {
// Allow up to MaxFuture second in the future blocks. If this limit
// is exceeded the chain is discarded and processed at a later time
// if given.
max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
if block.Time().Cmp(max) > 0 {
return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
case err == consensus.ErrFutureBlock:
// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
// the chain is discarded and processed at a later time if given.
max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
if block.Time().Cmp(max) > 0 {
return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
}
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
case err == consensus.ErrPrunedAncestor:
// Block competing with the canonical chain, store in the db, but don't process
// until the competitor TD goes above the canonical TD
localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64())
externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
if localTd.Cmp(externTd) > 0 {
if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
return i, events, coalescedLogs, err
}
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
}
// Competitor chain beat canonical, gather all blocks from the common ancestor
var winner []*types.Block
if err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()) {
bc.futureBlocks.Add(block.Hash(), block)
stats.queued++
continue
parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
for !bc.HasState(parent.Root()) {
winner = append(winner, parent)
parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
}
for j := 0; j < len(winner)/2; j++ {
winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
}
// Import all the pruned blocks to make the state available
bc.chainmu.Unlock()
_, evs, logs, err := bc.insertChain(winner)
bc.chainmu.Lock()
events, coalescedLogs = evs, logs
if err != nil {
return i, events, coalescedLogs, err
}
case err != nil:
bc.reportBlock(block, nil, err)
return i, events, coalescedLogs, err
}
@ -979,8 +1149,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
bc.reportBlock(block, receipts, err)
return i, events, coalescedLogs, err
}
proctime := time.Since(bstart)
// Write the block to the chain and get the status.
status, err := bc.WriteBlockAndState(block, receipts, state)
status, err := bc.WriteBlockWithState(block, receipts, state)
if err != nil {
return i, events, coalescedLogs, err
}
@ -994,6 +1166,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
events = append(events, ChainEvent{block, block.Hash(), logs})
lastCanon = block
// Only count canonical blocks for GC processing time
bc.gcproc += proctime
case SideStatTy:
log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
@ -1003,10 +1178,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
}
stats.processed++
stats.usedGas += usedGas
stats.report(chain, i)
stats.report(chain, i, bc.stateCache.TrieDB().Size())
}
// Append a single chain head event if we've progressed the chain
if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() {
if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
events = append(events, ChainHeadEvent{lastCanon})
}
return 0, events, coalescedLogs, nil
@ -1026,7 +1201,7 @@ const statsReportLimit = 8 * time.Second
// report prints statistics if some number of blocks have been processed
// or more than a few seconds have passed since the last message.
func (st *insertStats) report(chain []*types.Block, index int) {
func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
// Fetch the timings for the batch
var (
now = mclock.Now()
@ -1041,7 +1216,7 @@ func (st *insertStats) report(chain []*types.Block, index int) {
context := []interface{}{
"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
"number", end.Number(), "hash", end.Hash(),
"number", end.Number(), "hash", end.Hash(), "cache", cache,
}
if st.queued > 0 {
context = append(context, []interface{}{"queued", st.queued}...)
@ -1077,7 +1252,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
// These logs are later announced as deleted.
collectLogs = func(h common.Hash) {
// Coalesce logs and set 'Removed'.
receipts := GetBlockReceipts(bc.chainDb, h, bc.hc.GetBlockNumber(h))
receipts := GetBlockReceipts(bc.db, h, bc.hc.GetBlockNumber(h))
for _, receipt := range receipts {
for _, log := range receipt.Logs {
del := *log
@ -1146,7 +1321,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
// insert the block in the canonical way, re-writing history
bc.insert(newChain[i])
// write lookup entries for hash based transaction/receipt searches
if err := WriteTxLookupEntries(bc.chainDb, newChain[i]); err != nil {
if err := WriteTxLookupEntries(bc.db, newChain[i]); err != nil {
return err
}
addedTxs = append(addedTxs, newChain[i].Transactions()...)
@ -1156,7 +1331,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
// When transactions get deleted from the database that means the
// receipts that were created in the fork must also be deleted
for _, tx := range diff {
DeleteTxLookupEntry(bc.chainDb, tx.Hash())
DeleteTxLookupEntry(bc.db, tx.Hash())
}
if len(deletedLogs) > 0 {
go bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs})
@ -1248,7 +1423,7 @@ Hash: 0x%x
Error: %v
##############################
`, bc.config, block.Number(), block.Hash(), receiptString, err))
`, bc.chainConfig, block.Number(), block.Hash(), receiptString, err))
}
// InsertHeaderChain attempts to insert the given header chain in to the local
@ -1355,7 +1530,7 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
}
// Config retrieves the blockchain's chain configuration.
func (bc *BlockChain) Config() *params.ChainConfig { return bc.config }
func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig }
// Engine retrieves the blockchain's consensus engine.
func (bc *BlockChain) Engine() consensus.Engine { return bc.engine }

145
core/blockchain_test.go
View file

@ -46,7 +46,7 @@ func newTestBlockChain(fake bool) *BlockChain {
if !fake {
engine = ethash.NewTester()
}
blockchain, err := NewBlockChain(db, gspec.Config, engine, vm.Config{})
blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
if err != nil {
panic(err)
}
@ -148,9 +148,9 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
return err
}
blockchain.mu.Lock()
WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash())))
WriteBlock(blockchain.chainDb, block)
statedb.CommitTo(blockchain.chainDb, false)
WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash())))
WriteBlock(blockchain.db, block)
statedb.Commit(false)
blockchain.mu.Unlock()
}
return nil
@ -166,8 +166,8 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
}
// Manually insert the header into the database, but don't reorganise (allows subsequent testing)
blockchain.mu.Lock()
WriteTd(blockchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash)))
WriteHeader(blockchain.chainDb, header)
WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash)))
WriteHeader(blockchain.db, header)
blockchain.mu.Unlock()
}
return nil
@ -186,9 +186,9 @@ func TestLastBlock(t *testing.T) {
bchain := newTestBlockChain(false)
defer bchain.Stop()
block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.chainDb, 0)[0]
block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0]
bchain.insert(block)
if block.Hash() != GetHeadBlockHash(bchain.chainDb) {
if block.Hash() != GetHeadBlockHash(bchain.db) {
t.Errorf("Write/Get HeadBlockHash failed")
}
}
@ -496,7 +496,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
}
// Create a new BlockChain and check that it rolled back the state.
ncm, err := NewBlockChain(bc.chainDb, bc.config, ethash.NewFaker(), vm.Config{})
ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{})
if err != nil {
t.Fatalf("failed to create new chain manager: %v", err)
}
@ -609,7 +609,7 @@ func TestFastVsFullChains(t *testing.T) {
// Import the chain as an archive node for the comparison baseline
archiveDb, _ := ethdb.NewMemDatabase()
gspec.MustCommit(archiveDb)
archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{})
archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer archive.Stop()
if n, err := archive.InsertChain(blocks); err != nil {
@ -618,7 +618,7 @@ func TestFastVsFullChains(t *testing.T) {
// Fast import the chain as a non-archive node to test
fastDb, _ := ethdb.NewMemDatabase()
gspec.MustCommit(fastDb)
fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{})
fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
@ -696,7 +696,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
archiveDb, _ := ethdb.NewMemDatabase()
gspec.MustCommit(archiveDb)
archive, _ := NewBlockChain(archiveDb, gspec.Config, ethash.NewFaker(), vm.Config{})
archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
if n, err := archive.InsertChain(blocks); err != nil {
t.Fatalf("failed to process block %d: %v", n, err)
}
@ -709,7 +709,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
// Import the chain as a non-archive node and ensure all pointers are updated
fastDb, _ := ethdb.NewMemDatabase()
gspec.MustCommit(fastDb)
fast, _ := NewBlockChain(fastDb, gspec.Config, ethash.NewFaker(), vm.Config{})
fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
@ -730,7 +730,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
lightDb, _ := ethdb.NewMemDatabase()
gspec.MustCommit(lightDb)
light, _ := NewBlockChain(lightDb, gspec.Config, ethash.NewFaker(), vm.Config{})
light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
if n, err := light.InsertHeaderChain(headers, 1); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
@ -799,7 +799,7 @@ func TestChainTxReorgs(t *testing.T) {
}
})
// Import the chain. This runs all block validation rules.
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
if i, err := blockchain.InsertChain(chain); err != nil {
t.Fatalf("failed to insert original chain[%d]: %v", i, err)
}
@ -870,7 +870,7 @@ func TestLogReorgs(t *testing.T) {
signer = types.NewEIP155Signer(gspec.Config.ChainId)
)
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer blockchain.Stop()
rmLogsCh := make(chan RemovedLogsEvent)
@ -917,7 +917,7 @@ func TestReorgSideEvent(t *testing.T) {
signer = types.NewEIP155Signer(gspec.Config.ChainId)
)
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer blockchain.Stop()
chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {})
@ -992,7 +992,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
bc := newTestBlockChain(true)
defer bc.Stop()
chain, _ := GenerateChain(bc.config, bc.genesisBlock, ethash.NewFaker(), bc.chainDb, 10, func(i int, gen *BlockGen) {})
chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {})
var pend sync.WaitGroup
pend.Add(len(chain))
@ -1003,14 +1003,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
// try to retrieve a block by its canonical hash and see if the block data can be retrieved.
for {
ch := GetCanonicalHash(bc.chainDb, block.NumberU64())
ch := GetCanonicalHash(bc.db, block.NumberU64())
if ch == (common.Hash{}) {
continue // busy wait for canonical hash to be written
}
if ch != block.Hash() {
t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex())
}
fb := GetBlock(bc.chainDb, ch, block.NumberU64())
fb := GetBlock(bc.db, ch, block.NumberU64())
if fb == nil {
t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex())
}
@ -1043,7 +1043,7 @@ func TestEIP155Transition(t *testing.T) {
genesis = gspec.MustCommit(db)
)
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer blockchain.Stop()
blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) {
@ -1151,7 +1151,7 @@ func TestEIP161AccountRemoval(t *testing.T) {
}
genesis = gspec.MustCommit(db)
)
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer blockchain.Stop()
blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) {
@ -1226,7 +1226,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
diskdb, _ := ethdb.NewMemDatabase()
new(Genesis).MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, params.TestChainConfig, engine, vm.Config{})
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
@ -1245,3 +1245,102 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
}
}
}
// Tests that importing small side forks doesn't leave junk in the trie database
// cache (which would eventually cause memory issues).
func TestTrieForkGC(t *testing.T) {
// Generate a canonical chain to act as the main dataset
engine := ethash.NewFaker()
db, _ := ethdb.NewMemDatabase()
genesis := new(Genesis).MustCommit(db)
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
// Generate a bunch of fork blocks, each side forking from the canonical chain
forks := make([]*types.Block, len(blocks))
for i := 0; i < len(forks); i++ {
parent := genesis
if i > 0 {
parent = blocks[i-1]
}
fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) })
forks[i] = fork[0]
}
// Import the canonical and fork chain side by side, forcing the trie cache to cache both
diskdb, _ := ethdb.NewMemDatabase()
new(Genesis).MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
for i := 0; i < len(blocks); i++ {
if _, err := chain.InsertChain(blocks[i : i+1]); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", i, err)
}
if _, err := chain.InsertChain(forks[i : i+1]); err != nil {
t.Fatalf("fork %d: failed to insert into chain: %v", i, err)
}
}
// Dereference all the recent tries and ensure no past trie is left in
for i := 0; i < triesInMemory; i++ {
chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root(), common.Hash{})
chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root(), common.Hash{})
}
if len(chain.stateCache.TrieDB().Nodes()) > 0 {
t.Fatalf("stale tries still alive after garbase collection")
}
}
// Tests that doing large reorgs works even if the state associated with the
// forking point is not available any more.
func TestLargeReorgTrieGC(t *testing.T) {
// Generate the original common chain segment and the two competing forks
engine := ethash.NewFaker()
db, _ := ethdb.NewMemDatabase()
genesis := new(Genesis).MustCommit(db)
shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) })
original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) })
competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*triesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
// Import the shared chain and the original canonical one
diskdb, _ := ethdb.NewMemDatabase()
new(Genesis).MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if _, err := chain.InsertChain(shared); err != nil {
t.Fatalf("failed to insert shared chain: %v", err)
}
if _, err := chain.InsertChain(original); err != nil {
t.Fatalf("failed to insert shared chain: %v", err)
}
// Ensure that the state associated with the forking point is pruned away
if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {
t.Fatalf("common-but-old ancestor still cache")
}
// Import the competitor chain without exceeding the canonical's TD and ensure
// we have not processed any of the blocks (protection against malicious blocks)
if _, err := chain.InsertChain(competitor[:len(competitor)-2]); err != nil {
t.Fatalf("failed to insert competitor chain: %v", err)
}
for i, block := range competitor[:len(competitor)-2] {
if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil {
t.Fatalf("competitor %d: low TD chain became processed", i)
}
}
// Import the head of the competitor chain, triggering the reorg and ensure we
// successfully reprocess all the stashed away blocks.
if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil {
t.Fatalf("failed to finalize competitor chain: %v", err)
}
for i, block := range competitor[:len(competitor)-triesInMemory] {
if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil {
t.Fatalf("competitor %d: competing chain state missing", i)
}
}
}

3
core/chain_indexer.go
View file

@ -203,6 +203,9 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE
if header.ParentHash != prevHash {
// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
// TODO(karalabe): This operation is expensive and might block, causing the event system to
// potentially also lock up. We need to do with on a different thread somehow.
if h := FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
c.newHead(h.Number.Uint64(), true)
}

9
core/chain_makers.go
View file

@ -166,7 +166,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
blockchain, _ := NewBlockChain(db, config, engine, vm.Config{})
blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{})
defer blockchain.Stop()
b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
@ -192,10 +192,13 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
if b.engine != nil {
block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
// Write state changes to db
_, err := statedb.CommitTo(db, config.IsEIP158(b.header.Number))
root, err := statedb.Commit(config.IsEIP158(b.header.Number))
if err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}
if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
panic(fmt.Sprintf("trie write error: %v", err))
}
return block, b.receipts
}
return nil, nil
@ -246,7 +249,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B
db, _ := ethdb.NewMemDatabase()
genesis := gspec.MustCommit(db)
blockchain, _ := NewBlockChain(db, params.AllEthashProtocolChanges, engine, vm.Config{})
blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{})
// Create and inject the requested chain
if n == 0 {
return db, blockchain, nil

2
core/chain_makers_test.go
View file

@ -79,7 +79,7 @@ func ExampleGenerateChain() {
})
// Import the chain. This runs all block validation rules.
blockchain, _ := NewBlockChain(db, gspec.Config, ethash.NewFaker(), vm.Config{})
blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
defer blockchain.Stop()
if i, err := blockchain.InsertChain(chain); err != nil {

24
core/dao_test.go
View file

@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
proConf.DAOForkBlock = forkBlock
proConf.DAOForkSupport = true
proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{})
proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{})
defer proBc.Stop()
conDb, _ := ethdb.NewMemDatabase()
@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
conConf.DAOForkBlock = forkBlock
conConf.DAOForkSupport = false
conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{})
conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{})
defer conBc.Stop()
if _, err := proBc.InsertChain(prefix); err != nil {
@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
// Create a pro-fork block, and try to feed into the no-fork chain
db, _ = ethdb.NewMemDatabase()
gspec.MustCommit(db)
bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{})
bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
defer bc.Stop()
blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@ -79,6 +79,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
if _, err := conBc.InsertChain(blocks); err == nil {
t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0])
@ -91,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
// Create a no-fork block, and try to feed into the pro-fork chain
db, _ = ethdb.NewMemDatabase()
gspec.MustCommit(db)
bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{})
bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
defer bc.Stop()
blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
@ -101,6 +104,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
if _, err := proBc.InsertChain(blocks); err == nil {
t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0])
@ -114,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
db, _ = ethdb.NewMemDatabase()
gspec.MustCommit(db)
bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{})
bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
defer bc.Stop()
blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@ -124,6 +130,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
if _, err := conBc.InsertChain(blocks); err != nil {
t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err)
@ -131,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
db, _ = ethdb.NewMemDatabase()
gspec.MustCommit(db)
bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{})
bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
defer bc.Stop()
blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
@ -141,6 +150,9 @@ func TestDAOForkRangeExtradata(t *testing.T) {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
}
if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil {
t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
}
blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {})
if _, err := proBc.InsertChain(blocks); err != nil {
t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err)

24
core/genesis.go
View file

@ -169,10 +169,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
// Check whether the genesis block is already written.
if genesis != nil {
block, _ := genesis.ToBlock()
hash := block.Hash()
hash := genesis.ToBlock(nil).Hash()
if hash != stored {
return genesis.Config, block.Hash(), &GenesisMismatchError{stored, hash}
return genesis.Config, hash, &GenesisMismatchError{stored, hash}
}
}
@ -220,9 +219,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
}
}
// ToBlock creates the block and state of a genesis specification.
func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
db, _ := ethdb.NewMemDatabase()
// ToBlock creates the genesis block and writes state of a genesis specification
// to the given database (or discards it if nil).
func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
if db == nil {
db, _ = ethdb.NewMemDatabase()
}
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
for addr, account := range g.Alloc {
statedb.AddBalance(addr, account.Balance)
@ -252,19 +254,19 @@ func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) {
if g.Difficulty == nil {
head.Difficulty = params.GenesisDifficulty
}
return types.NewBlock(head, nil, nil, nil), statedb
statedb.Commit(false)
statedb.Database().TrieDB().Commit(root, true)
return types.NewBlock(head, nil, nil, nil)
}
// Commit writes the block and state of a genesis specification to the database.
// The block is committed as the canonical head block.
func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
block, statedb := g.ToBlock()
block := g.ToBlock(db)
if block.Number().Sign() != 0 {
return nil, fmt.Errorf("can't commit genesis block with number > 0")
}
if _, err := statedb.CommitTo(db, false); err != nil {
return nil, fmt.Errorf("cannot write state: %v", err)
}
if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil {
return nil, err
}

6
core/genesis_test.go
View file

@ -30,11 +30,11 @@ import (
)
func TestDefaultGenesisBlock(t *testing.T) {
block, _ := DefaultGenesisBlock().ToBlock()
block := DefaultGenesisBlock().ToBlock(nil)
if block.Hash() != params.MainnetGenesisHash {
t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash)
}
block, _ = DefaultTestnetGenesisBlock().ToBlock()
block = DefaultTestnetGenesisBlock().ToBlock(nil)
if block.Hash() != params.TestnetGenesisHash {
t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash)
}
@ -118,7 +118,7 @@ func TestSetupGenesis(t *testing.T) {
// Commit the 'old' genesis block with Homestead transition at #2.
// Advance to block #4, past the homestead transition block of customg.
genesis := oldcustomg.MustCommit(db)
bc, _ := NewBlockChain(db, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
defer bc.Stop()
bc.SetValidator(bproc{})
bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0))

51
core/state/database.go
View file

@ -40,16 +40,23 @@ const (
// Database wraps access to tries and contract code.
type Database interface {
// Accessing tries:
// OpenTrie opens the main account trie.
// OpenStorageTrie opens the storage trie of an account.
OpenTrie(root common.Hash) (Trie, error)
// OpenStorageTrie opens the storage trie of an account.
OpenStorageTrie(addrHash, root common.Hash) (Trie, error)
// Accessing contract code:
ContractCode(addrHash, codeHash common.Hash) ([]byte, error)
ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
// ContractCode retrieves a particular contract's code.
ContractCode(addrHash, codeHash common.Hash) ([]byte, error)
// ContractCodeSize retrieves a particular contracts code's size.
ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
// TrieDB retrieves the low level trie database used for data storage.
TrieDB() *trie.Database
}
// Trie is a Ethereum Merkle Trie.
@ -57,26 +64,33 @@ type Trie interface {
TryGet(key []byte) ([]byte, error)
TryUpdate(key, value []byte) error
TryDelete(key []byte) error
CommitTo(trie.DatabaseWriter) (common.Hash, error)
Commit(onleaf trie.LeafCallback) (common.Hash, error)
Hash() common.Hash
NodeIterator(startKey []byte) trie.NodeIterator
GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed
Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error
}
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains cached trie nodes in memory.
// concurrent use and retains cached trie nodes in memory. The pool is an optional
// intermediate trie-node memory pool between the low level storage layer and the
// high level trie abstraction.
func NewDatabase(db ethdb.Database) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{db: db, codeSizeCache: csc}
return &cachingDB{
db: trie.NewDatabase(db),
codeSizeCache: csc,
}
}
type cachingDB struct {
db ethdb.Database
db *trie.Database
mu sync.Mutex
pastTries []*trie.SecureTrie
codeSizeCache *lru.Cache
}
// OpenTrie opens the main account trie.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
db.mu.Lock()
defer db.mu.Unlock()
@ -105,10 +119,12 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
}
}
// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
return trie.NewSecure(root, db.db, 0)
}
// CopyTrie returns an independent copy of the given trie.
func (db *cachingDB) CopyTrie(t Trie) Trie {
switch t := t.(type) {
case cachedTrie:
@ -120,14 +136,16 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
}
}
// ContractCode retrieves a particular contract's code.
func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
code, err := db.db.Get(codeHash[:])
code, err := db.db.Node(codeHash)
if err == nil {
db.codeSizeCache.Add(codeHash, len(code))
}
return code, err
}
// ContractCodeSize retrieves a particular contracts code's size.
func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) {
if cached, ok := db.codeSizeCache.Get(codeHash); ok {
return cached.(int), nil
@ -139,16 +157,25 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
return len(code), err
}
// TrieDB retrieves any intermediate trie-node caching layer.
func (db *cachingDB) TrieDB() *trie.Database {
return db.db
}
// cachedTrie inserts its trie into a cachingDB on commit.
type cachedTrie struct {
*trie.SecureTrie
db *cachingDB
}
func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
root, err := m.SecureTrie.CommitTo(dbw)
func (m cachedTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
root, err := m.SecureTrie.Commit(onleaf)
if err == nil {
m.db.pushTrie(m.SecureTrie)
}
return root, err
}
func (m cachedTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
return m.SecureTrie.Prove(key, fromLevel, proofDb)
}

17
core/state/iterator_test.go
View file

@ -21,12 +21,13 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
)
// Tests that the node iterator indeed walks over the entire database contents.
func TestNodeIteratorCoverage(t *testing.T) {
// Create some arbitrary test state to iterate
db, mem, root, _ := makeTestState()
db, root, _ := makeTestState()
state, err := New(root, db)
if err != nil {
@ -39,14 +40,18 @@ func TestNodeIteratorCoverage(t *testing.T) {
hashes[it.Hash] = struct{}{}
}
}
// Cross check the hashes and the database itself
// Cross check the iterated hashes and the database/nodepool content
for hash := range hashes {
if _, err := mem.Get(hash.Bytes()); err != nil {
t.Errorf("failed to retrieve reported node %x: %v", hash, err)
if _, err := db.TrieDB().Node(hash); err != nil {
t.Errorf("failed to retrieve reported node %x", hash)
}
}
for _, key := range mem.Keys() {
for _, hash := range db.TrieDB().Nodes() {
if _, ok := hashes[hash]; !ok {
t.Errorf("state entry not reported %x", hash)
}
}
for _, key := range db.TrieDB().DiskDB().(*ethdb.MemDatabase).Keys() {
if bytes.HasPrefix(key, []byte("secure-key-")) {
continue
}

5
core/state/state_object.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
var emptyCodeHash = crypto.Keccak256(nil)
@ -238,12 +237,12 @@ func (self *stateObject) updateRoot(db Database) {
// CommitTrie the storage trie of the object to dwb.
// This updates the trie root.
func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error {
func (self *stateObject) CommitTrie(db Database) error {
self.updateTrie(db)
if self.dbErr != nil {
return self.dbErr
}
root, err := self.trie.CommitTo(dbw)
root, err := self.trie.Commit(nil)
if err == nil {
self.data.Root = root
}

6
core/state/state_test.go
View file

@ -48,7 +48,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
// write some of them to the trie
s.state.updateStateObject(obj1)
s.state.updateStateObject(obj2)
s.state.CommitTo(s.db, false)
s.state.Commit(false)
// check that dump contains the state objects that are in trie
got := string(s.state.Dump())
@ -97,7 +97,7 @@ func (s *StateSuite) TestNull(c *checker.C) {
//value := common.FromHex("0x823140710bf13990e4500136726d8b55")
var value common.Hash
s.state.SetState(address, common.Hash{}, value)
s.state.CommitTo(s.db, false)
s.state.Commit(false)
value = s.state.GetState(address, common.Hash{})
if !common.EmptyHash(value) {
c.Errorf("expected empty hash. got %x", value)
@ -155,7 +155,7 @@ func TestSnapshot2(t *testing.T) {
so0.deleted = false
state.setStateObject(so0)
root, _ := state.CommitTo(db, false)
root, _ := state.Commit(false)
state.Reset(root)
// and one with deleted == true

38
core/state/statedb.go
View file

@ -36,6 +36,14 @@ type revision struct {
journalIndex int
}
var (
// emptyState is the known hash of an empty state trie entry.
emptyState = crypto.Keccak256Hash(nil)
// emptyCode is the known hash of the empty EVM bytecode.
emptyCode = crypto.Keccak256Hash(nil)
)
// StateDBs within the ethereum protocol are used to store anything
// within the merkle trie. StateDBs take care of caching and storing
// nested states. It's the general query interface to retrieve:
@ -235,6 +243,11 @@ func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
return common.Hash{}
}
// Database retrieves the low level database supporting the lower level trie ops.
func (self *StateDB) Database() Database {
return self.db
}
// StorageTrie returns the storage trie of an account.
// The return value is a copy and is nil for non-existent accounts.
func (self *StateDB) StorageTrie(a common.Address) Trie {
@ -568,8 +581,8 @@ func (s *StateDB) clearJournalAndRefund() {
s.refund = 0
}
// CommitTo writes the state to the given database.
func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root common.Hash, err error) {
// Commit writes the state to the underlying in-memory trie database.
func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
defer s.clearJournalAndRefund()
// Commit objects to the trie.
@ -583,13 +596,11 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
case isDirty:
// Write any contract code associated with the state object
if stateObject.code != nil && stateObject.dirtyCode {
if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil {
return common.Hash{}, err
}
s.db.TrieDB().Insert(common.BytesToHash(stateObject.CodeHash()), stateObject.code)
stateObject.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie.
if err := stateObject.CommitTrie(s.db, dbw); err != nil {
if err := stateObject.CommitTrie(s.db); err != nil {
return common.Hash{}, err
}
// Update the object in the main account trie.
@ -598,7 +609,20 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
delete(s.stateObjectsDirty, addr)
}
// Write trie changes.
root, err = s.trie.CommitTo(dbw)
root, err = s.trie.Commit(func(leaf []byte, parent common.Hash) error {
var account Account
if err := rlp.DecodeBytes(leaf, &account); err != nil {
return nil
}
if account.Root != emptyState {
s.db.TrieDB().Reference(account.Root, parent)
}
code := common.BytesToHash(account.CodeHash)
if code != emptyCode {
s.db.TrieDB().Reference(code, parent)
}
return nil
})
log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads())
return root, err
}

14
core/state/statedb_test.go
View file

@ -97,10 +97,10 @@ func TestIntermediateLeaks(t *testing.T) {
}
// Commit and cross check the databases.
if _, err := transState.CommitTo(transDb, false); err != nil {
if _, err := transState.Commit(false); err != nil {
t.Fatalf("failed to commit transition state: %v", err)
}
if _, err := finalState.CommitTo(finalDb, false); err != nil {
if _, err := finalState.Commit(false); err != nil {
t.Fatalf("failed to commit final state: %v", err)
}
for _, key := range finalDb.Keys() {
@ -122,8 +122,8 @@ func TestIntermediateLeaks(t *testing.T) {
// https://github.com/ethereum/go-ethereum/pull/15549.
func TestCopy(t *testing.T) {
// Create a random state test to copy and modify "independently"
mem, _ := ethdb.NewMemDatabase()
orig, _ := New(common.Hash{}, NewDatabase(mem))
db, _ := ethdb.NewMemDatabase()
orig, _ := New(common.Hash{}, NewDatabase(db))
for i := byte(0); i < 255; i++ {
obj := orig.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
@ -346,11 +346,10 @@ func (test *snapshotTest) run() bool {
}
action.fn(action, state)
}
// Revert all snapshots in reverse order. Each revert must yield a state
// that is equivalent to fresh state with all actions up the snapshot applied.
for sindex--; sindex >= 0; sindex-- {
checkstate, _ := New(common.Hash{}, NewDatabase(db))
checkstate, _ := New(common.Hash{}, state.Database())
for _, action := range test.actions[:test.snapshots[sindex]] {
action.fn(action, checkstate)
}
@ -409,7 +408,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error {
func (s *StateSuite) TestTouchDelete(c *check.C) {
s.state.GetOrNewStateObject(common.Address{})
root, _ := s.state.CommitTo(s.db, false)
root, _ := s.state.Commit(false)
s.state.Reset(root)
snapshot := s.state.Snapshot()
@ -417,7 +416,6 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
if len(s.state.stateObjectsDirty) != 1 {
c.Fatal("expected one dirty state object")
}
s.state.RevertToSnapshot(snapshot)
if len(s.state.stateObjectsDirty) != 0 {
c.Fatal("expected no dirty state object")

44
core/state/sync_test.go
View file

@ -36,10 +36,10 @@ type testAccount struct {
}
// makeTestState create a sample test state to test node-wise reconstruction.
func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) {
func makeTestState() (Database, common.Hash, []*testAccount) {
// Create an empty state
mem, _ := ethdb.NewMemDatabase()
db := NewDatabase(mem)
diskdb, _ := ethdb.NewMemDatabase()
db := NewDatabase(diskdb)
state, _ := New(common.Hash{}, db)
// Fill it with some arbitrary data
@ -61,10 +61,10 @@ func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount)
state.updateStateObject(obj)
accounts = append(accounts, acc)
}
root, _ := state.CommitTo(mem, false)
root, _ := state.Commit(false)
// Return the generated state
return db, mem, root, accounts
return db, root, accounts
}
// checkStateAccounts cross references a reconstructed state with an expected
@ -96,7 +96,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
if v, _ := db.Get(root[:]); v == nil {
return nil // Consider a non existent state consistent.
}
trie, err := trie.New(root, db)
trie, err := trie.New(root, trie.NewDatabase(db))
if err != nil {
return err
}
@ -138,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t,
func testIterativeStateSync(t *testing.T, batch int) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -148,9 +148,9 @@ func testIterativeStateSync(t *testing.T, batch int) {
for len(queue) > 0 {
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcMem.Get(hash.Bytes())
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
@ -170,7 +170,7 @@ func testIterativeStateSync(t *testing.T, batch int) {
// partial results are returned, and the others sent only later.
func TestIterativeDelayedStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -181,9 +181,9 @@ func TestIterativeDelayedStateSync(t *testing.T) {
// Sync only half of the scheduled nodes
results := make([]trie.SyncResult, len(queue)/2+1)
for i, hash := range queue[:len(results)] {
data, err := srcMem.Get(hash.Bytes())
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
@ -207,7 +207,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomS
func testIterativeRandomStateSync(t *testing.T, batch int) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -221,9 +221,9 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
// Fetch all the queued nodes in a random order
results := make([]trie.SyncResult, 0, len(queue))
for hash := range queue {
data, err := srcMem.Get(hash.Bytes())
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
t.Fatalf("failed to retrieve node data for %x", hash)
}
results = append(results, trie.SyncResult{Hash: hash, Data: data})
}
@ -247,7 +247,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
// partial results are returned (Even those randomly), others sent only later.
func TestIterativeRandomDelayedStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcRoot, srcAccounts := makeTestState()
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -263,9 +263,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
for hash := range queue {
delete(queue, hash)
data, err := srcMem.Get(hash.Bytes())
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
t.Fatalf("failed to retrieve node data for %x", hash)
}
results = append(results, trie.SyncResult{Hash: hash, Data: data})
@ -292,9 +292,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
// the database.
func TestIncompleteStateSync(t *testing.T) {
// Create a random state to copy
_, srcMem, srcRoot, srcAccounts := makeTestState()
srcDb, srcRoot, srcAccounts := makeTestState()
checkTrieConsistency(srcMem, srcRoot)
checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot)
// Create a destination state and sync with the scheduler
dstDb, _ := ethdb.NewMemDatabase()
@ -306,9 +306,9 @@ func TestIncompleteStateSync(t *testing.T) {
// Fetch a batch of state nodes
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcMem.Get(hash.Bytes())
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}

3
core/state_transition.go
View file

@ -215,6 +215,9 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
// Pay intrinsic gas
gas, err := IntrinsicGas(st.data, contractCreation, homestead)
if err != nil {
return nil, 0, false, err
}
if err = st.useGas(gas); err != nil {
return nil, 0, false, err
}

4
core/tx_pool_test.go
View file

@ -78,8 +78,8 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec
}
func setupTxPool() (*TxPool, *ecdsa.PrivateKey) {
db, _ := ethdb.NewMemDatabase()
statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
diskdb, _ := ethdb.NewMemDatabase()
statedb, _ := state.New(common.Hash{}, state.NewDatabase(diskdb))
blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)}
key, _ := crypto.GenerateKey()

9
core/types/block.go
View file

@ -25,6 +25,7 @@ import (
"sort"
"sync/atomic"
"time"
"unsafe"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -121,6 +122,12 @@ func (h *Header) HashNoNonce() common.Hash {
})
}
// Size returns the approximate memory used by all internal contents. It is used
// to approximate and limit the memory consumption of various caches.
func (h *Header) Size() common.StorageSize {
return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8)
}
func rlpHash(x interface{}) (h common.Hash) {
hw := sha3.NewKeccak256()
rlp.Encode(hw, x)
@ -322,6 +329,8 @@ func (b *Block) HashNoNonce() common.Hash {
return b.header.HashNoNonce()
}
// Size returns the true RLP encoded storage size of the block, either by encoding
// and returning it, or returning a previsouly cached value.
func (b *Block) Size() common.StorageSize {
if size := b.size.Load(); size != nil {
return size.(common.StorageSize)

13
core/types/receipt.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"fmt"
"io"
"unsafe"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -136,6 +137,18 @@ func (r *Receipt) statusEncoding() []byte {
return r.PostState
}
// Size returns the approximate memory used by all internal contents. It is used
// to approximate and limit the memory consumption of various caches.
func (r *Receipt) Size() common.StorageSize {
size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState))
size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{}))
for _, log := range r.Logs {
size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data))
}
return size
}
// String implements the Stringer interface.
func (r *Receipt) String() string {
if len(r.PostState) == 0 {

2
core/types/transaction.go
View file

@ -206,6 +206,8 @@ func (tx *Transaction) Hash() common.Hash {
return v
}
// Size returns the true RLP encoded storage size of the transaction, either by
// encoding and returning it, or returning a previsouly cached value.
func (tx *Transaction) Size() common.StorageSize {
if size := tx.size.Load(); size != nil {
return size.(common.StorageSize)

16
core/vm/contracts_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package vm
import (

16
core/vm/instructions_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package vm
import (

7
core/vm/interpreter.go
View file

@ -20,9 +20,7 @@ import (
"fmt"
"sync/atomic"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
)
@ -123,11 +121,6 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
return nil, nil
}
codehash := contract.CodeHash // codehash is used when doing jump dest caching
if codehash == (common.Hash{}) {
codehash = crypto.Keccak256Hash(contract.Code)
}
var (
op OpCode // current opcode
mem = NewMemory() // bound memory

4
eth/api.go
View file

@ -462,11 +462,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
oldTrie, err := trie.NewSecure(startBlock.Root(), api.eth.chainDb, 0)
oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
if err != nil {
return nil, err
}
newTrie, err := trie.NewSecure(endBlock.Root(), api.eth.chainDb, 0)
newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb), 0)
if err != nil {
return nil, err
}

2
eth/api_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify

135
eth/api_tracer.go
View file

@ -24,7 +24,6 @@ import (
"io/ioutil"
"runtime"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
@ -34,7 +33,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/tracers"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
@ -72,6 +70,7 @@ type txTraceResult struct {
type blockTraceTask struct {
statedb *state.StateDB // Intermediate state prepped for tracing
block *types.Block // Block to trace the transactions from
rootref common.Hash // Trie root reference held for this task
results []*txTraceResult // Trace results procudes by the task
}
@ -90,59 +89,6 @@ type txTraceTask struct {
index int // Transaction offset in the block
}
// ephemeralDatabase is a memory wrapper around a proper database, which acts as
// an ephemeral write layer. This construct is used by the chain tracer to write
// state tries for intermediate blocks without serializing to disk, but at the
// same time to allow disk fallback for reads that do no hit the memory layer.
type ephemeralDatabase struct {
diskdb ethdb.Database // Persistent disk database to fall back to with reads
memdb *ethdb.MemDatabase // Ephemeral memory database for primary reads and writes
}
func (db *ephemeralDatabase) Put(key []byte, value []byte) error { return db.memdb.Put(key, value) }
func (db *ephemeralDatabase) Delete(key []byte) error { return errors.New("delete not supported") }
func (db *ephemeralDatabase) Close() { db.memdb.Close() }
func (db *ephemeralDatabase) NewBatch() ethdb.Batch {
return db.memdb.NewBatch()
}
func (db *ephemeralDatabase) Has(key []byte) (bool, error) {
if has, _ := db.memdb.Has(key); has {
return has, nil
}
return db.diskdb.Has(key)
}
func (db *ephemeralDatabase) Get(key []byte) ([]byte, error) {
if blob, _ := db.memdb.Get(key); blob != nil {
return blob, nil
}
return db.diskdb.Get(key)
}
// Prune does a state sync into a new memory write layer and replaces the old one.
// This allows us to discard entries that are no longer referenced from the current
// state.
func (db *ephemeralDatabase) Prune(root common.Hash) {
// Pull the still relevant state data into memory
sync := state.NewStateSync(root, db.diskdb)
for sync.Pending() > 0 {
hash := sync.Missing(1)[0]
// Move the next trie node from the memory layer into a sync struct
node, err := db.memdb.Get(hash[:])
if err != nil {
panic(err) // memdb must have the data
}
if _, _, err := sync.Process([]trie.SyncResult{{Hash: hash, Data: node}}); err != nil {
panic(err) // it's not possible to fail processing a node
}
}
// Discard the old memory layer and write a new one
db.memdb, _ = ethdb.NewMemDatabaseWithCap(db.memdb.Len())
if _, err := sync.Commit(db); err != nil {
panic(err) // writing into a memdb cannot fail
}
}
// TraceChain returns the structured logs created during the execution of EVM
// between two blocks (excluding start) and returns them as a JSON object.
func (api *PrivateDebugAPI) TraceChain(ctx context.Context, start, end rpc.BlockNumber, config *TraceConfig) (*rpc.Subscription, error) {
@ -188,19 +134,15 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
// Ensure we have a valid starting state before doing any work
origin := start.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
memdb, _ := ethdb.NewMemDatabase()
db := &ephemeralDatabase{
diskdb: api.eth.ChainDb(),
memdb: memdb,
}
if number := start.NumberU64(); number > 0 {
start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
if start == nil {
return nil, fmt.Errorf("parent block #%d not found", number-1)
}
}
statedb, err := state.New(start.Root(), state.NewDatabase(db))
statedb, err := state.New(start.Root(), database)
if err != nil {
// If the starting state is missing, allow some number of blocks to be reexecuted
reexec := defaultTraceReexec
@ -213,7 +155,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
if start == nil {
break
}
if statedb, err = state.New(start.Root(), state.NewDatabase(db)); err == nil {
if statedb, err = state.New(start.Root(), database); err == nil {
break
}
}
@ -256,7 +198,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
res, err := api.traceTx(ctx, msg, vmctx, task.statedb, config)
if err != nil {
task.results[i] = &txTraceResult{Error: err.Error()}
log.Warn("Tracing failed", "err", err)
log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err)
break
}
task.statedb.DeleteSuicides()
@ -273,7 +215,6 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
}
// Start a goroutine to feed all the blocks into the tracers
begin := time.Now()
complete := start.NumberU64()
go func() {
var (
@ -281,6 +222,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
number uint64
traced uint64
failed error
proot common.Hash
)
// Ensure everything is properly cleaned up on any exit path
defer func() {
@ -308,7 +250,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
// Print progress logs if long enough time elapsed
if time.Since(logged) > 8*time.Second {
if number > origin {
log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin))
log.Info("Tracing chain segment", "start", origin, "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin), "memory", database.TrieDB().Size())
} else {
log.Info("Preparing state for chain trace", "block", number, "start", origin, "elapsed", time.Since(begin))
}
@ -325,13 +267,11 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
txs := block.Transactions()
select {
case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: block, results: make([]*txTraceResult, len(txs))}:
case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: block, rootref: proot, results: make([]*txTraceResult, len(txs))}:
case <-notifier.Closed():
return
}
traced += uint64(len(txs))
} else {
atomic.StoreUint64(&complete, number)
}
// Generate the next state snapshot fast without tracing
_, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{})
@ -340,7 +280,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
break
}
// Finalize the state so any modifications are written to the trie
root, err := statedb.CommitTo(db, true)
root, err := statedb.Commit(true)
if err != nil {
failed = err
break
@ -349,26 +289,14 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
failed = err
break
}
// After every N blocks, prune the database to only retain relevant data
if (number-start.NumberU64())%4096 == 0 {
// Wait until currently pending trace jobs finish
for atomic.LoadUint64(&complete) != number {
select {
case <-time.After(100 * time.Millisecond):
case <-notifier.Closed():
return
}
}
// No more concurrent access at this point, prune the database
var (
nodes = db.memdb.Len()
start = time.Now()
)
db.Prune(root)
log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(start))
statedb, _ = state.New(root, state.NewDatabase(db))
// Reference the trie twice, once for us, once for the trancer
database.TrieDB().Reference(root, common.Hash{})
if number >= origin {
database.TrieDB().Reference(root, common.Hash{})
}
// Dereference all past tries we ourselves are done working with
database.TrieDB().Dereference(proot, common.Hash{})
proot = root
}
}()
@ -387,12 +315,14 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
}
done[uint64(result.Block)] = result
// Dereference any paret tries held in memory by this task
database.TrieDB().Dereference(res.rootref, common.Hash{})
// Stream completed traces to the user, aborting on the first error
for result, ok := done[next]; ok; result, ok = done[next] {
if len(result.Traces) > 0 || next == end.NumberU64() {
notifier.Notify(sub.ID, result)
}
atomic.StoreUint64(&complete, next)
delete(done, next)
next++
}
@ -544,18 +474,14 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
}
// Otherwise try to reexec blocks until we find a state or reach our limit
origin := block.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
memdb, _ := ethdb.NewMemDatabase()
db := &ephemeralDatabase{
diskdb: api.eth.ChainDb(),
memdb: memdb,
}
for i := uint64(0); i < reexec; i++ {
block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
if block == nil {
break
}
if statedb, err = state.New(block.Root(), state.NewDatabase(db)); err == nil {
if statedb, err = state.New(block.Root(), database); err == nil {
break
}
}
@ -571,6 +497,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
var (
start = time.Now()
logged time.Time
proot common.Hash
)
for block.NumberU64() < origin {
// Print progress logs if long enough time elapsed
@ -587,26 +514,18 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
return nil, err
}
// Finalize the state so any modifications are written to the trie
root, err := statedb.CommitTo(db, true)
root, err := statedb.Commit(true)
if err != nil {
return nil, err
}
if err := statedb.Reset(root); err != nil {
return nil, err
}
// After every N blocks, prune the database to only retain relevant data
if block.NumberU64()%4096 == 0 || block.NumberU64() == origin {
var (
nodes = db.memdb.Len()
begin = time.Now()
)
db.Prune(root)
log.Info("Pruned tracer state entries", "deleted", nodes-db.memdb.Len(), "left", db.memdb.Len(), "elapsed", time.Since(begin))
statedb, _ = state.New(root, state.NewDatabase(db))
}
database.TrieDB().Reference(root, common.Hash{})
database.TrieDB().Dereference(proot, common.Hash{})
proot = root
}
log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start))
log.Info("Historical state regenerated", "block", block.NumberU64(), "elapsed", time.Since(start), "size", database.TrieDB().Size())
return statedb, nil
}

14
eth/backend.go
View file

@ -144,9 +144,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
}
core.WriteBlockChainVersion(chainDb, core.BlockChainVersion)
}
vmConfig := vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
eth.blockchain, err = core.NewBlockChain(chainDb, eth.chainConfig, eth.engine, vmConfig)
var (
vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
)
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, eth.chainConfig, eth.engine, vmConfig)
if err != nil {
return nil, err
}
@ -393,10 +395,10 @@ func (s *Ethereum) Start(srvr *p2p.Server) error {
// Figure out a max peers count based on the server limits
maxPeers := srvr.MaxPeers
if s.config.LightServ > 0 {
maxPeers -= s.config.LightPeers
if maxPeers < srvr.MaxPeers/2 {
maxPeers = srvr.MaxPeers / 2
if s.config.LightPeers >= srvr.MaxPeers {
return fmt.Errorf("invalid peer config: light peer count (%d) >= total peer count (%d)", s.config.LightPeers, srvr.MaxPeers)
}
maxPeers -= s.config.LightPeers
}
// Start the networking layer and the light server if requested
s.protocolManager.Start(maxPeers)

10
eth/config.go
View file

@ -22,6 +22,7 @@ import (
"os/user"
"path/filepath"
"runtime"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -43,8 +44,10 @@ var DefaultConfig = Config{
DatasetsOnDisk: 2,
},
NetworkId: 1,
LightPeers: 20,
DatabaseCache: 128,
LightPeers: 100,
DatabaseCache: 768,
TrieCache: 256,
TrieTimeout: 5 * time.Minute,
GasPrice: big.NewInt(18 * params.Shannon),
TxPool: core.DefaultTxPoolConfig,
@ -78,6 +81,7 @@ type Config struct {
// Protocol options
NetworkId uint64 // Network ID to use for selecting peers to connect to
SyncMode downloader.SyncMode
NoPruning bool
// Light client options
LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests
@ -87,6 +91,8 @@ type Config struct {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
TrieCache int
TrieTimeout time.Duration
// Mining-related options
Etherbase common.Address `toml:",omitempty"`

356
eth/downloader/downloader.go
View file

@ -18,10 +18,8 @@
package downloader
import (
"crypto/rand"
"errors"
"fmt"
"math"
"math/big"
"sync"
"sync/atomic"
@ -61,12 +59,11 @@ var (
maxHeadersProcess = 2048 // Number of header download results to import at once into the chain
maxResultsProcess = 2048 // Number of content download results to import at once into the chain
fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
fsPivotInterval = 256 // Number of headers out of which to randomize the pivot point
fsMinFullBlocks = 64 // Number of blocks to retrieve fully even in fast sync
fsCriticalTrials = uint32(32) // Number of times to retry in the cricical section before bailing
fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
fsHeaderContCheck = 3 * time.Second // Time interval to check for header continuations during state download
fsMinFullBlocks = 64 // Number of blocks to retrieve fully even in fast sync
)
var (
@ -102,9 +99,6 @@ type Downloader struct {
peers *peerSet // Set of active peers from which download can proceed
stateDB ethdb.Database
fsPivotLock *types.Header // Pivot header on critical section entry (cannot change between retries)
fsPivotFails uint32 // Number of subsequent fast sync failures in the critical section
rttEstimate uint64 // Round trip time to target for download requests
rttConfidence uint64 // Confidence in the estimated RTT (unit: millionths to allow atomic ops)
@ -124,6 +118,7 @@ type Downloader struct {
synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing
synchronising int32
notified int32
committed int32
// Channels
headerCh chan dataPack // [eth/62] Channel receiving inbound block headers
@ -156,7 +151,7 @@ type Downloader struct {
// LightChain encapsulates functions required to synchronise a light chain.
type LightChain interface {
// HasHeader verifies a header's presence in the local chain.
HasHeader(h common.Hash, number uint64) bool
HasHeader(common.Hash, uint64) bool
// GetHeaderByHash retrieves a header from the local chain.
GetHeaderByHash(common.Hash) *types.Header
@ -164,8 +159,8 @@ type LightChain interface {
// CurrentHeader retrieves the head header from the local chain.
CurrentHeader() *types.Header
// GetTdByHash returns the total difficulty of a local block.
GetTdByHash(common.Hash) *big.Int
// GetTd returns the total difficulty of a local block.
GetTd(common.Hash, uint64) *big.Int
// InsertHeaderChain inserts a batch of headers into the local chain.
InsertHeaderChain([]*types.Header, int) (int, error)
@ -178,8 +173,8 @@ type LightChain interface {
type BlockChain interface {
LightChain
// HasBlockAndState verifies block and associated states' presence in the local chain.
HasBlockAndState(common.Hash) bool
// HasBlock verifies a block's presence in the local chain.
HasBlock(common.Hash, uint64) bool
// GetBlockByHash retrieves a block from the local chain.
GetBlockByHash(common.Hash) *types.Block
@ -271,7 +266,6 @@ func (d *Downloader) Synchronising() bool {
// RegisterPeer injects a new download peer into the set of block source to be
// used for fetching hashes and blocks from.
func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error {
logger := log.New("peer", id)
logger.Trace("Registering sync peer")
if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil {
@ -324,8 +318,13 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode
errEmptyHeaderSet, errPeersUnavailable, errTooOld,
errInvalidAncestor, errInvalidChain:
log.Warn("Synchronisation failed, dropping peer", "peer", id, "err", err)
d.dropPeer(id)
if d.dropPeer == nil {
// The dropPeer method is nil when `--copydb` is used for a local copy.
// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", id)
} else {
d.dropPeer(id)
}
default:
log.Warn("Synchronisation failed, retrying", "err", err)
}
@ -386,9 +385,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode
// Set the requested sync mode, unless it's forbidden
d.mode = mode
if d.mode == FastSync && atomic.LoadUint32(&d.fsPivotFails) >= fsCriticalTrials {
d.mode = FullSync
}
// Retrieve the origin peer and initiate the downloading process
p := d.peers.Peer(id)
if p == nil {
@ -436,57 +433,40 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
d.syncStatsChainHeight = height
d.syncStatsLock.Unlock()
// Initiate the sync using a concurrent header and content retrieval algorithm
// Ensure our origin point is below any fast sync pivot point
pivot := uint64(0)
switch d.mode {
case LightSync:
pivot = height
case FastSync:
// Calculate the new fast/slow sync pivot point
if d.fsPivotLock == nil {
pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval)))
if err != nil {
panic(fmt.Sprintf("Failed to access crypto random source: %v", err))
}
if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() {
pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64()
}
if d.mode == FastSync {
if height <= uint64(fsMinFullBlocks) {
origin = 0
} else {
// Pivot point locked in, use this and do not pick a new one!
pivot = d.fsPivotLock.Number.Uint64()
}
// If the point is below the origin, move origin back to ensure state download
if pivot < origin {
if pivot > 0 {
pivot = height - uint64(fsMinFullBlocks)
if pivot <= origin {
origin = pivot - 1
} else {
origin = 0
}
}
log.Debug("Fast syncing until pivot block", "pivot", pivot)
}
d.queue.Prepare(origin+1, d.mode, pivot, latest)
d.committed = 1
if d.mode == FastSync && pivot != 0 {
d.committed = 0
}
// Initiate the sync using a concurrent header and content retrieval algorithm
d.queue.Prepare(origin+1, d.mode)
if d.syncInitHook != nil {
d.syncInitHook(origin, height)
}
fetchers := []func() error{
func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved
func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync
func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync
func() error { return d.processHeaders(origin+1, td) },
func() error { return d.fetchHeaders(p, origin+1, pivot) }, // Headers are always retrieved
func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync
func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync
func() error { return d.processHeaders(origin+1, pivot, td) },
}
if d.mode == FastSync {
fetchers = append(fetchers, func() error { return d.processFastSyncContent(latest) })
} else if d.mode == FullSync {
fetchers = append(fetchers, d.processFullSyncContent)
}
err = d.spawnSync(fetchers)
if err != nil && d.mode == FastSync && d.fsPivotLock != nil {
// If sync failed in the critical section, bump the fail counter.
atomic.AddUint32(&d.fsPivotFails, 1)
}
return err
return d.spawnSync(fetchers)
}
// spawnSync runs d.process and all given fetcher functions to completion in
@ -602,7 +582,6 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
// Figure out the valid ancestor range to prevent rewrite attacks
floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64()
p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
if d.mode == FullSync {
ceil = d.blockchain.CurrentBlock().NumberU64()
} else if d.mode == FastSync {
@ -611,6 +590,8 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
if ceil >= MaxForkAncestry {
floor = int64(ceil - MaxForkAncestry)
}
p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height)
// Request the topmost blocks to short circuit binary ancestor lookup
head := ceil
if head > height {
@ -666,7 +647,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
continue
}
// Otherwise check if we already know the header or not
if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
if (d.mode == FullSync && d.blockchain.HasBlock(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
number, hash = headers[i].Number.Uint64(), headers[i].Hash()
// If every header is known, even future ones, the peer straight out lied about its head
@ -731,7 +712,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
arrived = true
// Modify the search interval based on the response
if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
if (d.mode == FullSync && !d.blockchain.HasBlock(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
end = check
break
}
@ -769,7 +750,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
// other peers are only accepted if they map cleanly to the skeleton. If no one
// can fill in the skeleton - not even the origin peer - it's assumed invalid and
// the origin is dropped.
func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64) error {
p.log.Debug("Directing header downloads", "origin", from)
defer p.log.Debug("Header download terminated")
@ -820,6 +801,18 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
}
// If no more headers are inbound, notify the content fetchers and return
if packet.Items() == 0 {
// Don't abort header fetches while the pivot is downloading
if atomic.LoadInt32(&d.committed) == 0 && pivot <= from {
p.log.Debug("No headers, waiting for pivot commit")
select {
case <-time.After(fsHeaderContCheck):
getHeaders(from)
continue
case <-d.cancelCh:
return errCancelHeaderFetch
}
}
// Pivot done (or not in fast sync) and no more headers, terminate the process
p.log.Debug("No more headers available")
select {
case d.headerProcCh <- nil:
@ -853,6 +846,12 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error {
getHeaders(from)
case <-timeout.C:
if d.dropPeer == nil {
// The dropPeer method is nil when `--copydb` is used for a local copy.
// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
p.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", p.id)
break
}
// Header retrieval timed out, consider the peer bad and drop
p.log.Debug("Header request timed out", "elapsed", ttl)
headerTimeoutMeter.Mark(1)
@ -1071,7 +1070,13 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
setIdle(peer, 0)
} else {
peer.log.Debug("Stalling delivery, dropping", "type", kind)
d.dropPeer(pid)
if d.dropPeer == nil {
// The dropPeer method is nil when `--copydb` is used for a local copy.
// Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored
peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", pid)
} else {
d.dropPeer(pid)
}
}
}
}
@ -1112,10 +1117,8 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
}
if request.From > 0 {
peer.log.Trace("Requesting new batch of data", "type", kind, "from", request.From)
} else if len(request.Headers) > 0 {
peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number)
} else {
peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Hashes))
peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number)
}
// Fetch the chunk and make sure any errors return the hashes to the queue
if fetchHook != nil {
@ -1143,10 +1146,7 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv
// processHeaders takes batches of retrieved headers from an input channel and
// keeps processing and scheduling them into the header chain and downloader's
// queue until the stream ends or a failure occurs.
func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
// Calculate the pivoting point for switching from fast to slow sync
pivot := d.queue.FastSyncPivot()
func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) error {
// Keep a count of uncertain headers to roll back
rollback := []*types.Header{}
defer func() {
@ -1171,19 +1171,6 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
"header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number),
"fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock),
"block", fmt.Sprintf("%d->%d", lastBlock, curBlock))
// If we're already past the pivot point, this could be an attack, thread carefully
if rollback[len(rollback)-1].Number.Uint64() > pivot {
// If we didn't ever fail, lock in the pivot header (must! not! change!)
if atomic.LoadUint32(&d.fsPivotFails) == 0 {
for _, header := range rollback {
if header.Number.Uint64() == pivot {
log.Warn("Fast-sync pivot locked in", "number", pivot, "hash", header.Hash())
d.fsPivotLock = header
}
}
}
}
}
}()
@ -1218,7 +1205,8 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
// L: Request new headers up from 11 (R's TD was higher, it must have something)
// R: Nothing to give
if d.mode != LightSync {
if !gotHeaders && td.Cmp(d.blockchain.GetTdByHash(d.blockchain.CurrentBlock().Hash())) > 0 {
head := d.blockchain.CurrentBlock()
if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 {
return errStallingPeer
}
}
@ -1230,7 +1218,8 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
// queued for processing when the header download completes. However, as long as the
// peer gave us something useful, we're already happy/progressed (above check).
if d.mode == FastSync || d.mode == LightSync {
if td.Cmp(d.lightchain.GetTdByHash(d.lightchain.CurrentHeader().Hash())) > 0 {
head := d.lightchain.CurrentHeader()
if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 {
return errStallingPeer
}
}
@ -1283,13 +1272,6 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
rollback = append(rollback[:0], rollback[len(rollback)-fsHeaderSafetyNet:]...)
}
}
// If we're fast syncing and just pulled in the pivot, make sure it's the one locked in
if d.mode == FastSync && d.fsPivotLock != nil && chunk[0].Number.Uint64() <= pivot && chunk[len(chunk)-1].Number.Uint64() >= pivot {
if pivot := chunk[int(pivot-chunk[0].Number.Uint64())]; pivot.Hash() != d.fsPivotLock.Hash() {
log.Warn("Pivot doesn't match locked in one", "remoteNumber", pivot.Number, "remoteHash", pivot.Hash(), "localNumber", d.fsPivotLock.Number, "localHash", d.fsPivotLock.Hash())
return errInvalidChain
}
}
// Unless we're doing light chains, schedule the headers for associated content retrieval
if d.mode == FullSync || d.mode == FastSync {
// If we've reached the allowed number of pending headers, stall a bit
@ -1324,7 +1306,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error {
// processFullSyncContent takes fetch results from the queue and imports them into the chain.
func (d *Downloader) processFullSyncContent() error {
for {
results := d.queue.WaitResults()
results := d.queue.Results(true)
if len(results) == 0 {
return nil
}
@ -1338,30 +1320,28 @@ func (d *Downloader) processFullSyncContent() error {
}
func (d *Downloader) importBlockResults(results []*fetchResult) error {
for len(results) != 0 {
// Check for any termination requests. This makes clean shutdown faster.
select {
case <-d.quitCh:
return errCancelContentProcessing
default:
}
// Retrieve the a batch of results to import
items := int(math.Min(float64(len(results)), float64(maxResultsProcess)))
first, last := results[0].Header, results[items-1].Header
log.Debug("Inserting downloaded chain", "items", len(results),
"firstnum", first.Number, "firsthash", first.Hash(),
"lastnum", last.Number, "lasthash", last.Hash(),
)
blocks := make([]*types.Block, items)
for i, result := range results[:items] {
blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
}
if index, err := d.blockchain.InsertChain(blocks); err != nil {
log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
return errInvalidChain
}
// Shift the results to the next batch
results = results[items:]
// Check for any early termination requests
if len(results) == 0 {
return nil
}
select {
case <-d.quitCh:
return errCancelContentProcessing
default:
}
// Retrieve the a batch of results to import
first, last := results[0].Header, results[len(results)-1].Header
log.Debug("Inserting downloaded chain", "items", len(results),
"firstnum", first.Number, "firsthash", first.Hash(),
"lastnum", last.Number, "lasthash", last.Hash(),
)
blocks := make([]*types.Block, len(results))
for i, result := range results {
blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
}
if index, err := d.blockchain.InsertChain(blocks); err != nil {
log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
return errInvalidChain
}
return nil
}
@ -1369,35 +1349,92 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error {
// processFastSyncContent takes fetch results from the queue and writes them to the
// database. It also controls the synchronisation of state nodes of the pivot block.
func (d *Downloader) processFastSyncContent(latest *types.Header) error {
// Start syncing state of the reported head block.
// This should get us most of the state of the pivot block.
// Start syncing state of the reported head block. This should get us most of
// the state of the pivot block.
stateSync := d.syncState(latest.Root)
defer stateSync.Cancel()
go func() {
if err := stateSync.Wait(); err != nil {
if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
d.queue.Close() // wake up WaitResults
}
}()
pivot := d.queue.FastSyncPivot()
// Figure out the ideal pivot block. Note, that this goalpost may move if the
// sync takes long enough for the chain head to move significantly.
pivot := uint64(0)
if height := latest.Number.Uint64(); height > uint64(fsMinFullBlocks) {
pivot = height - uint64(fsMinFullBlocks)
}
// To cater for moving pivot points, track the pivot block and subsequently
// accumulated download results separatey.
var (
oldPivot *fetchResult // Locked in pivot block, might change eventually
oldTail []*fetchResult // Downloaded content after the pivot
)
for {
results := d.queue.WaitResults()
// Wait for the next batch of downloaded data to be available, and if the pivot
// block became stale, move the goalpost
results := d.queue.Results(oldPivot == nil) // Block if we're not monitoring pivot staleness
if len(results) == 0 {
return stateSync.Cancel()
// If pivot sync is done, stop
if oldPivot == nil {
return stateSync.Cancel()
}
// If sync failed, stop
select {
case <-d.cancelCh:
return stateSync.Cancel()
default:
}
}
if d.chainInsertHook != nil {
d.chainInsertHook(results)
}
if oldPivot != nil {
results = append(append([]*fetchResult{oldPivot}, oldTail...), results...)
}
// Split around the pivot block and process the two sides via fast/full sync
if atomic.LoadInt32(&d.committed) == 0 {
latest = results[len(results)-1].Header
if height := latest.Number.Uint64(); height > pivot+2*uint64(fsMinFullBlocks) {
log.Warn("Pivot became stale, moving", "old", pivot, "new", height-uint64(fsMinFullBlocks))
pivot = height - uint64(fsMinFullBlocks)
}
}
P, beforeP, afterP := splitAroundPivot(pivot, results)
if err := d.commitFastSyncData(beforeP, stateSync); err != nil {
return err
}
if P != nil {
stateSync.Cancel()
if err := d.commitPivotBlock(P); err != nil {
return err
// If new pivot block found, cancel old state retrieval and restart
if oldPivot != P {
stateSync.Cancel()
stateSync = d.syncState(P.Header.Root)
defer stateSync.Cancel()
go func() {
if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
d.queue.Close() // wake up WaitResults
}
}()
oldPivot = P
}
// Wait for completion, occasionally checking for pivot staleness
select {
case <-stateSync.done:
if stateSync.err != nil {
return stateSync.err
}
if err := d.commitPivotBlock(P); err != nil {
return err
}
oldPivot = nil
case <-time.After(time.Second):
oldTail = afterP
continue
}
}
// Fast sync done, pivot commit done, full import
if err := d.importBlockResults(afterP); err != nil {
return err
}
@ -1420,52 +1457,49 @@ func splitAroundPivot(pivot uint64, results []*fetchResult) (p *fetchResult, bef
}
func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *stateSync) error {
for len(results) != 0 {
// Check for any termination requests.
select {
case <-d.quitCh:
return errCancelContentProcessing
case <-stateSync.done:
if err := stateSync.Wait(); err != nil {
return err
}
default:
// Check for any early termination requests
if len(results) == 0 {
return nil
}
select {
case <-d.quitCh:
return errCancelContentProcessing
case <-stateSync.done:
if err := stateSync.Wait(); err != nil {
return err
}
// Retrieve the a batch of results to import
items := int(math.Min(float64(len(results)), float64(maxResultsProcess)))
first, last := results[0].Header, results[items-1].Header
log.Debug("Inserting fast-sync blocks", "items", len(results),
"firstnum", first.Number, "firsthash", first.Hash(),
"lastnumn", last.Number, "lasthash", last.Hash(),
)
blocks := make([]*types.Block, items)
receipts := make([]types.Receipts, items)
for i, result := range results[:items] {
blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
receipts[i] = result.Receipts
}
if index, err := d.blockchain.InsertReceiptChain(blocks, receipts); err != nil {
log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
return errInvalidChain
}
// Shift the results to the next batch
results = results[items:]
default:
}
// Retrieve the a batch of results to import
first, last := results[0].Header, results[len(results)-1].Header
log.Debug("Inserting fast-sync blocks", "items", len(results),
"firstnum", first.Number, "firsthash", first.Hash(),
"lastnumn", last.Number, "lasthash", last.Hash(),
)
blocks := make([]*types.Block, len(results))
receipts := make([]types.Receipts, len(results))
for i, result := range results {
blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
receipts[i] = result.Receipts
}
if index, err := d.blockchain.InsertReceiptChain(blocks, receipts); err != nil {
log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
return errInvalidChain
}
return nil
}
func (d *Downloader) commitPivotBlock(result *fetchResult) error {
b := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
// Sync the pivot block state. This should complete reasonably quickly because
// we've already synced up to the reported head block state earlier.
if err := d.syncState(b.Root()).Wait(); err != nil {
block := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles)
log.Debug("Committing fast sync pivot as new head", "number", block.Number(), "hash", block.Hash())
if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{result.Receipts}); err != nil {
return err
}
log.Debug("Committing fast sync pivot as new head", "number", b.Number(), "hash", b.Hash())
if _, err := d.blockchain.InsertReceiptChain([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil {
if err := d.blockchain.FastSyncCommitHead(block.Hash()); err != nil {
return err
}
return d.blockchain.FastSyncCommitHead(b.Hash())
atomic.StoreInt32(&d.committed, 1)
return nil
}
// DeliverHeaders injects a new batch of block headers received from a remote

207
eth/downloader/downloader_test.go
View file

@ -28,7 +28,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
@ -45,8 +44,8 @@ var (
// Reduce some of the parameters to make the tester faster.
func init() {
MaxForkAncestry = uint64(10000)
blockCacheLimit = 1024
fsCriticalTrials = 10
blockCacheItems = 1024
fsHeaderContCheck = 500 * time.Millisecond
}
// downloadTester is a test simulator for mocking out local block chain.
@ -222,14 +221,9 @@ func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool {
return dl.GetHeaderByHash(hash) != nil
}
// HasBlockAndState checks if a block and associated state is present in the testers canonical chain.
func (dl *downloadTester) HasBlockAndState(hash common.Hash) bool {
block := dl.GetBlockByHash(hash)
if block == nil {
return false
}
_, err := dl.stateDb.Get(block.Root().Bytes())
return err == nil
// HasBlock checks if a block is present in the testers canonical chain.
func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool {
return dl.GetBlockByHash(hash) != nil
}
// GetHeader retrieves a header from the testers canonical chain.
@ -293,14 +287,14 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block {
func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error {
// For now only check that the state trie is correct
if block := dl.GetBlockByHash(hash); block != nil {
_, err := trie.NewSecure(block.Root(), dl.stateDb, 0)
_, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb), 0)
return err
}
return fmt.Errorf("non existent block: %x", hash[:4])
}
// GetTdByHash retrieves the block's total difficulty from the canonical chain.
func (dl *downloadTester) GetTdByHash(hash common.Hash) *big.Int {
// GetTd retrieves the block's total difficulty from the canonical chain.
func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int {
dl.lock.RLock()
defer dl.lock.RUnlock()
@ -619,28 +613,22 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
// number of items of the various chain components.
func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
// Initialize the counters for the first fork
headers, blocks := lengths[0], lengths[0]
headers, blocks, receipts := lengths[0], lengths[0], lengths[0]-fsMinFullBlocks
minReceipts, maxReceipts := lengths[0]-fsMinFullBlocks-fsPivotInterval, lengths[0]-fsMinFullBlocks
if minReceipts < 0 {
minReceipts = 1
}
if maxReceipts < 0 {
maxReceipts = 1
if receipts < 0 {
receipts = 1
}
// Update the counters for each subsequent fork
for _, length := range lengths[1:] {
headers += length - common
blocks += length - common
minReceipts += length - common - fsMinFullBlocks - fsPivotInterval
maxReceipts += length - common - fsMinFullBlocks
receipts += length - common - fsMinFullBlocks
}
switch tester.downloader.mode {
case FullSync:
minReceipts, maxReceipts = 1, 1
receipts = 1
case LightSync:
blocks, minReceipts, maxReceipts = 1, 1, 1
blocks, receipts = 1, 1
}
if hs := len(tester.ownHeaders); hs != headers {
t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers)
@ -648,11 +636,12 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
if bs := len(tester.ownBlocks); bs != blocks {
t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks)
}
if rs := len(tester.ownReceipts); rs < minReceipts || rs > maxReceipts {
t.Fatalf("synchronised receipts mismatch: have %v, want between [%v, %v]", rs, minReceipts, maxReceipts)
if rs := len(tester.ownReceipts); rs != receipts {
t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts)
}
// Verify the state trie too for fast syncs
if tester.downloader.mode == FastSync {
/*if tester.downloader.mode == FastSync {
pivot := uint64(0)
var index int
if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common {
index = pivot
@ -660,11 +649,11 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng
index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
}
if index > 0 {
if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb)); statedb == nil || err != nil {
if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(trie.NewDatabase(tester.stateDb))); statedb == nil || err != nil {
t.Fatalf("state reconstruction failed: %v", err)
}
}
}
}*/
}
// Tests that simple synchronization against a canonical chain works correctly.
@ -684,7 +673,7 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@ -710,7 +699,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a long block chain to download and the tester
targetBlocks := 8 * blockCacheLimit
targetBlocks := 8 * blockCacheItems
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@ -745,9 +734,9 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
cached = len(tester.downloader.queue.blockDonePool)
if mode == FastSync {
if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached {
if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
cached = receipts
}
//if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
cached = receipts
//}
}
}
frozen = int(atomic.LoadUint32(&blocked))
@ -755,7 +744,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
tester.downloader.queue.lock.Unlock()
tester.lock.Unlock()
if cached == blockCacheLimit || retrieved+cached+frozen == targetBlocks+1 {
if cached == blockCacheItems || retrieved+cached+frozen == targetBlocks+1 {
break
}
}
@ -765,8 +754,8 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
tester.lock.RLock()
retrieved = len(tester.ownBlocks)
tester.lock.RUnlock()
if cached != blockCacheLimit && retrieved+cached+frozen != targetBlocks+1 {
t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheLimit, retrieved, frozen, targetBlocks+1)
if cached != blockCacheItems && retrieved+cached+frozen != targetBlocks+1 {
t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheItems, retrieved, frozen, targetBlocks+1)
}
// Permit the blocked blocks to import
if atomic.LoadUint32(&blocked) > 0 {
@ -974,7 +963,7 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download and the tester
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
if targetBlocks >= MaxHashFetch {
targetBlocks = MaxHashFetch - 15
}
@ -1016,12 +1005,12 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Create various peers with various parts of the chain
targetPeers := 8
targetBlocks := targetPeers*blockCacheLimit - 15
targetBlocks := targetPeers*blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
for i := 0; i < targetPeers; i++ {
id := fmt.Sprintf("peer #%d", i)
tester.newPeer(id, protocol, hashes[i*blockCacheLimit:], headers, blocks, receipts)
tester.newPeer(id, protocol, hashes[i*blockCacheItems:], headers, blocks, receipts)
}
if err := tester.sync("peer #0", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
@ -1045,7 +1034,7 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Create peers of every type
@ -1084,7 +1073,7 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a block chain to download
targetBlocks := 2*blockCacheLimit - 15
targetBlocks := 2*blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
@ -1110,8 +1099,8 @@ func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
bodiesNeeded++
}
}
for hash, receipt := range receipts {
if mode == FastSync && len(receipt) > 0 && headers[hash].Number.Uint64() <= tester.downloader.queue.fastSyncPivot {
for _, receipt := range receipts {
if mode == FastSync && len(receipt) > 0 {
receiptsNeeded++
}
}
@ -1139,7 +1128,7 @@ func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Attempt a full sync with an attacker feeding gapped headers
@ -1174,7 +1163,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Attempt a full sync with an attacker feeding shifted headers
@ -1208,7 +1197,7 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := 3*fsHeaderSafetyNet + fsPivotInterval + fsMinFullBlocks
targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Attempt to sync with an attacker that feeds junk during the fast sync phase.
@ -1248,7 +1237,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
tester.downloader.fsPivotFails = 0
tester.downloader.syncInitHook = func(uint64, uint64) {
for i := missing; i <= len(hashes); i++ {
delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
@ -1267,8 +1255,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
t.Errorf("fast sync pivot block #%d not rolled back", head)
}
}
tester.downloader.fsPivotFails = fsCriticalTrials
// Synchronise with the valid peer and make sure sync succeeds. Since the last
// rollback should also disable fast syncing for this process, verify that we
// did a fresh full sync. Note, we can't assert anything about the receipts
@ -1383,7 +1369,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Set a sync init hook to catch progress changes
@ -1532,7 +1518,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Set a sync init hook to catch progress changes
@ -1609,7 +1595,7 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
defer tester.terminate()
// Create a small block chain
targetBlocks := blockCacheLimit - 15
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks+3, 0, tester.genesis, nil, false)
// Set a sync init hook to catch progress changes
@ -1697,6 +1683,7 @@ func TestDeliverHeadersHang(t *testing.T) {
type floodingTestPeer struct {
peer Peer
tester *downloadTester
pend sync.WaitGroup
}
func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() }
@ -1717,9 +1704,12 @@ func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int
deliveriesDone := make(chan struct{}, 500)
for i := 0; i < cap(deliveriesDone); i++ {
peer := fmt.Sprintf("fake-peer%d", i)
ftp.pend.Add(1)
go func() {
ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}})
deliveriesDone <- struct{}{}
ftp.pend.Done()
}()
}
// Deliver the actual requested headers.
@ -1751,110 +1741,15 @@ func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) {
// Whenever the downloader requests headers, flood it with
// a lot of unrequested header deliveries.
tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{
tester.downloader.peers.peers["peer"].peer,
tester,
peer: tester.downloader.peers.peers["peer"].peer,
tester: tester,
}
if err := tester.sync("peer", nil, mode); err != nil {
t.Errorf("sync failed: %v", err)
t.Errorf("test %d: sync failed: %v", i, err)
}
tester.terminate()
// Flush all goroutines to prevent messing with subsequent tests
tester.downloader.peers.peers["peer"].peer.(*floodingTestPeer).pend.Wait()
}
}
// Tests that if fast sync aborts in the critical section, it can restart a few
// times before giving up.
// We use data driven subtests to manage this so that it will be parallel on its own
// and not with the other tests, avoiding intermittent failures.
func TestFastCriticalRestarts(t *testing.T) {
testCases := []struct {
protocol int
progress bool
}{
{63, false},
{64, false},
{63, true},
{64, true},
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("protocol %d progress %v", tc.protocol, tc.progress), func(t *testing.T) {
testFastCriticalRestarts(t, tc.protocol, tc.progress)
})
}
}
func testFastCriticalRestarts(t *testing.T, protocol int, progress bool) {
t.Parallel()
tester := newTester()
defer tester.terminate()
// Create a large enough blockchin to actually fast sync on
targetBlocks := fsMinFullBlocks + 2*fsPivotInterval - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Create a tester peer with a critical section header missing (force failures)
tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
delete(tester.peerHeaders["peer"], hashes[fsMinFullBlocks-1])
tester.downloader.dropPeer = func(id string) {} // We reuse the same "faulty" peer throughout the test
// Remove all possible pivot state roots and slow down replies (test failure resets later)
for i := 0; i < fsPivotInterval; i++ {
tester.peerMissingStates["peer"][headers[hashes[fsMinFullBlocks+i]].Root] = true
}
(tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).setDelay(500 * time.Millisecond) // Enough to reach the critical section
// Synchronise with the peer a few times and make sure they fail until the retry limit
for i := 0; i < int(fsCriticalTrials)-1; i++ {
// Attempt a sync and ensure it fails properly
if err := tester.sync("peer", nil, FastSync); err == nil {
t.Fatalf("failing fast sync succeeded: %v", err)
}
time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
// If it's the first failure, pivot should be locked => reenable all others to detect pivot changes
if i == 0 {
time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
if tester.downloader.fsPivotLock == nil {
time.Sleep(400 * time.Millisecond) // Make sure the first huge timeout expires too
t.Fatalf("pivot block not locked in after critical section failure")
}
tester.lock.Lock()
tester.peerHeaders["peer"][hashes[fsMinFullBlocks-1]] = headers[hashes[fsMinFullBlocks-1]]
tester.peerMissingStates["peer"] = map[common.Hash]bool{tester.downloader.fsPivotLock.Root: true}
(tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).setDelay(0)
tester.lock.Unlock()
}
}
// Return all nodes if we're testing fast sync progression
if progress {
tester.lock.Lock()
tester.peerMissingStates["peer"] = map[common.Hash]bool{}
tester.lock.Unlock()
if err := tester.sync("peer", nil, FastSync); err != nil {
t.Fatalf("failed to synchronise blocks in progressed fast sync: %v", err)
}
time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
if fails := atomic.LoadUint32(&tester.downloader.fsPivotFails); fails != 1 {
t.Fatalf("progressed pivot trial count mismatch: have %v, want %v", fails, 1)
}
assertOwnChain(t, tester, targetBlocks+1)
} else {
if err := tester.sync("peer", nil, FastSync); err == nil {
t.Fatalf("succeeded to synchronise blocks in failed fast sync")
}
time.Sleep(150 * time.Millisecond) // Make sure no in-flight requests remain
if fails := atomic.LoadUint32(&tester.downloader.fsPivotFails); fails != fsCriticalTrials {
t.Fatalf("failed pivot trial count mismatch: have %v, want %v", fails, fsCriticalTrials)
}
}
// Retry limit exhausted, downloader will switch to full sync, should succeed
if err := tester.sync("peer", nil, FastSync); err != nil {
t.Fatalf("failed to synchronise blocks in slow sync: %v", err)
}
// Note, we can't assert the chain here because the test asserter assumes sync
// completed using a single mode of operation, whereas fast-then-slow can result
// in arbitrary intermediate state that's not cleanly verifiable.
}

171
eth/downloader/queue.go
View file

@ -32,7 +32,11 @@ import (
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)
var blockCacheLimit = 8192 // Maximum number of blocks to cache before throttling the download
var (
blockCacheItems = 8192 // Maximum number of blocks to cache before throttling the download
blockCacheMemory = 64 * 1024 * 1024 // Maximum amount of memory to use for block caching
blockCacheSizeWeight = 0.1 // Multiplier to approximate the average block size based on past ones
)
var (
errNoFetchesPending = errors.New("no fetches pending")
@ -41,17 +45,17 @@ var (
// fetchRequest is a currently running data retrieval operation.
type fetchRequest struct {
Peer *peerConnection // Peer to which the request was sent
From uint64 // [eth/62] Requested chain element index (used for skeleton fills only)
Hashes map[common.Hash]int // [eth/61] Requested hashes with their insertion index (priority)
Headers []*types.Header // [eth/62] Requested headers, sorted by request order
Time time.Time // Time when the request was made
Peer *peerConnection // Peer to which the request was sent
From uint64 // [eth/62] Requested chain element index (used for skeleton fills only)
Headers []*types.Header // [eth/62] Requested headers, sorted by request order
Time time.Time // Time when the request was made
}
// fetchResult is a struct collecting partial results from data fetchers until
// all outstanding pieces complete and the result as a whole can be processed.
type fetchResult struct {
Pending int // Number of data fetches still pending
Pending int // Number of data fetches still pending
Hash common.Hash // Hash of the header to prevent recalculating
Header *types.Header
Uncles []*types.Header
@ -61,12 +65,10 @@ type fetchResult struct {
// queue represents hashes that are either need fetching or are being fetched
type queue struct {
mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching
fastSyncPivot uint64 // Block number where the fast sync pivots into archive synchronisation mode
headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching
// Headers are "special", they download in batches, supported by a skeleton chain
headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
headerTaskPool map[uint64]*types.Header // [eth/62] Pending header retrieval tasks, mapping starting indexes to skeleton headers
headerTaskQueue *prque.Prque // [eth/62] Priority queue of the skeleton indexes to fetch the filling headers for
headerPeerMiss map[string]map[uint64]struct{} // [eth/62] Set of per-peer header batches known to be unavailable
@ -87,8 +89,9 @@ type queue struct {
receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations
receiptDonePool map[common.Hash]struct{} // [eth/63] Set of the completed receipt fetches
resultCache []*fetchResult // Downloaded but not yet delivered fetch results
resultOffset uint64 // Offset of the first cached fetch result in the block chain
resultCache []*fetchResult // Downloaded but not yet delivered fetch results
resultOffset uint64 // Offset of the first cached fetch result in the block chain
resultSize common.StorageSize // Approximate size of a block (exponential moving average)
lock *sync.Mutex
active *sync.Cond
@ -109,7 +112,7 @@ func newQueue() *queue {
receiptTaskQueue: prque.New(),
receiptPendPool: make(map[string]*fetchRequest),
receiptDonePool: make(map[common.Hash]struct{}),
resultCache: make([]*fetchResult, blockCacheLimit),
resultCache: make([]*fetchResult, blockCacheItems),
active: sync.NewCond(lock),
lock: lock,
}
@ -122,10 +125,8 @@ func (q *queue) Reset() {
q.closed = false
q.mode = FullSync
q.fastSyncPivot = 0
q.headerHead = common.Hash{}
q.headerPendPool = make(map[string]*fetchRequest)
q.blockTaskPool = make(map[common.Hash]*types.Header)
@ -138,7 +139,7 @@ func (q *queue) Reset() {
q.receiptPendPool = make(map[string]*fetchRequest)
q.receiptDonePool = make(map[common.Hash]struct{})
q.resultCache = make([]*fetchResult, blockCacheLimit)
q.resultCache = make([]*fetchResult, blockCacheItems)
q.resultOffset = 0
}
@ -214,27 +215,13 @@ func (q *queue) Idle() bool {
return (queued + pending + cached) == 0
}
// FastSyncPivot retrieves the currently used fast sync pivot point.
func (q *queue) FastSyncPivot() uint64 {
q.lock.Lock()
defer q.lock.Unlock()
return q.fastSyncPivot
}
// ShouldThrottleBlocks checks if the download should be throttled (active block (body)
// fetches exceed block cache).
func (q *queue) ShouldThrottleBlocks() bool {
q.lock.Lock()
defer q.lock.Unlock()
// Calculate the currently in-flight block (body) requests
pending := 0
for _, request := range q.blockPendPool {
pending += len(request.Hashes) + len(request.Headers)
}
// Throttle if more blocks (bodies) are in-flight than free space in the cache
return pending >= len(q.resultCache)-len(q.blockDonePool)
return q.resultSlots(q.blockPendPool, q.blockDonePool) <= 0
}
// ShouldThrottleReceipts checks if the download should be throttled (active receipt
@ -243,13 +230,39 @@ func (q *queue) ShouldThrottleReceipts() bool {
q.lock.Lock()
defer q.lock.Unlock()
// Calculate the currently in-flight receipt requests
pending := 0
for _, request := range q.receiptPendPool {
pending += len(request.Headers)
return q.resultSlots(q.receiptPendPool, q.receiptDonePool) <= 0
}
// resultSlots calculates the number of results slots available for requests
// whilst adhering to both the item and the memory limit too of the results
// cache.
func (q *queue) resultSlots(pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}) int {
// Calculate the maximum length capped by the memory limit
limit := len(q.resultCache)
if common.StorageSize(len(q.resultCache))*q.resultSize > common.StorageSize(blockCacheMemory) {
limit = int((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize)
}
// Throttle if more receipts are in-flight than free space in the cache
return pending >= len(q.resultCache)-len(q.receiptDonePool)
// Calculate the number of slots already finished
finished := 0
for _, result := range q.resultCache[:limit] {
if result == nil {
break
}
if _, ok := donePool[result.Hash]; ok {
finished++
}
}
// Calculate the number of slots currently downloading
pending := 0
for _, request := range pendPool {
for _, header := range request.Headers {
if header.Number.Uint64() < q.resultOffset+uint64(limit) {
pending++
}
}
}
// Return the free slots to distribute
return limit - finished - pending
}
// ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill
@ -323,8 +336,7 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
q.blockTaskPool[hash] = header
q.blockTaskQueue.Push(header, -float32(header.Number.Uint64()))
if q.mode == FastSync && header.Number.Uint64() <= q.fastSyncPivot {
// Fast phase of the fast sync, retrieve receipts too
if q.mode == FastSync {
q.receiptTaskPool[hash] = header
q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64()))
}
@ -335,18 +347,25 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
return inserts
}
// WaitResults retrieves and permanently removes a batch of fetch
// results from the cache. the result slice will be empty if the queue
// has been closed.
func (q *queue) WaitResults() []*fetchResult {
// Results retrieves and permanently removes a batch of fetch results from
// the cache. the result slice will be empty if the queue has been closed.
func (q *queue) Results(block bool) []*fetchResult {
q.lock.Lock()
defer q.lock.Unlock()
// Count the number of items available for processing
nproc := q.countProcessableItems()
for nproc == 0 && !q.closed {
if !block {
return nil
}
q.active.Wait()
nproc = q.countProcessableItems()
}
// Since we have a batch limit, don't pull more into "dangling" memory
if nproc > maxResultsProcess {
nproc = maxResultsProcess
}
results := make([]*fetchResult, nproc)
copy(results, q.resultCache[:nproc])
if len(results) > 0 {
@ -363,6 +382,21 @@ func (q *queue) WaitResults() []*fetchResult {
}
// Advance the expected block number of the first cache entry.
q.resultOffset += uint64(nproc)
// Recalculate the result item weights to prevent memory exhaustion
for _, result := range results {
size := result.Header.Size()
for _, uncle := range result.Uncles {
size += uncle.Size()
}
for _, receipt := range result.Receipts {
size += receipt.Size()
}
for _, tx := range result.Transactions {
size += tx.Size()
}
q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize
}
}
return results
}
@ -370,21 +404,9 @@ func (q *queue) WaitResults() []*fetchResult {
// countProcessableItems counts the processable items.
func (q *queue) countProcessableItems() int {
for i, result := range q.resultCache {
// Don't process incomplete or unavailable items.
if result == nil || result.Pending > 0 {
return i
}
// Stop before processing the pivot block to ensure that
// resultCache has space for fsHeaderForceVerify items. Not
// doing this could leave us unable to download the required
// amount of headers.
if q.mode == FastSync && result.Header.Number.Uint64() == q.fastSyncPivot {
for j := 0; j < fsHeaderForceVerify; j++ {
if i+j+1 >= len(q.resultCache) || q.resultCache[i+j+1] == nil {
return i
}
}
}
}
return len(q.resultCache)
}
@ -473,10 +495,8 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
return nil, false, nil
}
// Calculate an upper limit on the items we might fetch (i.e. throttling)
space := len(q.resultCache) - len(donePool)
for _, request := range pendPool {
space -= len(request.Headers)
}
space := q.resultSlots(pendPool, donePool)
// Retrieve a batch of tasks, skipping previously failed ones
send := make([]*types.Header, 0, count)
skip := make([]*types.Header, 0)
@ -484,6 +504,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
progress := false
for proc := 0; proc < space && len(send) < count && !taskQueue.Empty(); proc++ {
header := taskQueue.PopItem().(*types.Header)
hash := header.Hash()
// If we're the first to request this task, initialise the result container
index := int(header.Number.Int64() - int64(q.resultOffset))
@ -493,18 +514,19 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
}
if q.resultCache[index] == nil {
components := 1
if q.mode == FastSync && header.Number.Uint64() <= q.fastSyncPivot {
if q.mode == FastSync {
components = 2
}
q.resultCache[index] = &fetchResult{
Pending: components,
Hash: hash,
Header: header,
}
}
// If this fetch task is a noop, skip this fetch operation
if isNoop(header) {
donePool[header.Hash()] = struct{}{}
delete(taskPool, header.Hash())
donePool[hash] = struct{}{}
delete(taskPool, hash)
space, proc = space-1, proc-1
q.resultCache[index].Pending--
@ -512,7 +534,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
continue
}
// Otherwise unless the peer is known not to have the data, add to the retrieve list
if p.Lacks(header.Hash()) {
if p.Lacks(hash) {
skip = append(skip, header)
} else {
send = append(send, header)
@ -565,9 +587,6 @@ func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool m
if request.From > 0 {
taskQueue.Push(request.From, -float32(request.From))
}
for hash, index := range request.Hashes {
taskQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
@ -640,18 +659,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
if request.From > 0 {
taskQueue.Push(request.From, -float32(request.From))
}
for hash, index := range request.Hashes {
taskQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
// Add the peer to the expiry report along the the number of failed requests
expirations := len(request.Hashes)
if expirations < len(request.Headers) {
expirations = len(request.Headers)
}
expiries[id] = expirations
expiries[id] = len(request.Headers)
}
}
// Remove the expired requests from the pending pool
@ -828,14 +840,16 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
failure = err
break
}
donePool[header.Hash()] = struct{}{}
hash := header.Hash()
donePool[hash] = struct{}{}
q.resultCache[index].Pending--
useful = true
accepted++
// Clean up a successful fetch
request.Headers[i] = nil
delete(taskPool, header.Hash())
delete(taskPool, hash)
}
// Return all failed or missing fetches to the queue
for _, header := range request.Headers {
@ -860,7 +874,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
// Prepare configures the result cache to allow accepting and caching inbound
// fetch results.
func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64, head *types.Header) {
func (q *queue) Prepare(offset uint64, mode SyncMode) {
q.lock.Lock()
defer q.lock.Unlock()
@ -868,6 +882,5 @@ func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64, head *types.
if q.resultOffset < offset {
q.resultOffset = offset
}
q.fastSyncPivot = pivot
q.mode = mode
}

31
eth/downloader/statesync.go
View file

@ -20,7 +20,6 @@ import (
"fmt"
"hash"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
@ -294,6 +293,9 @@ func (s *stateSync) loop() error {
case <-s.cancel:
return errCancelStateFetch
case <-s.d.cancelCh:
return errCancelStateFetch
case req := <-s.deliver:
// Response, disconnect or timeout triggered, drop the peer if stalling
log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut())
@ -304,15 +306,11 @@ func (s *stateSync) loop() error {
s.d.dropPeer(req.peer.id)
}
// Process all the received blobs and check for stale delivery
stale, err := s.process(req)
if err != nil {
if err := s.process(req); err != nil {
log.Warn("Node data write error", "err", err)
return err
}
// The the delivery contains requested data, mark the node idle (otherwise it's a timed out delivery)
if !stale {
req.peer.SetNodeDataIdle(len(req.response))
}
req.peer.SetNodeDataIdle(len(req.response))
}
}
return s.commit(true)
@ -352,6 +350,7 @@ func (s *stateSync) assignTasks() {
case s.d.trackStateReq <- req:
req.peer.FetchNodeData(req.items)
case <-s.cancel:
case <-s.d.cancelCh:
}
}
}
@ -390,7 +389,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) {
// process iterates over a batch of delivered state data, injecting each item
// into a running state sync, re-queuing any items that were requested but not
// delivered.
func (s *stateSync) process(req *stateReq) (bool, error) {
func (s *stateSync) process(req *stateReq) error {
// Collect processing stats and update progress if valid data was received
duplicate, unexpected := 0, 0
@ -401,7 +400,7 @@ func (s *stateSync) process(req *stateReq) (bool, error) {
}(time.Now())
// Iterate over all the delivered data and inject one-by-one into the trie
progress, stale := false, len(req.response) > 0
progress := false
for _, blob := range req.response {
prog, hash, err := s.processNodeData(blob)
@ -415,20 +414,12 @@ func (s *stateSync) process(req *stateReq) (bool, error) {
case trie.ErrAlreadyProcessed:
duplicate++
default:
return stale, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
}
// If the node delivered a requested item, mark the delivery non-stale
if _, ok := req.tasks[hash]; ok {
delete(req.tasks, hash)
stale = false
}
}
// If we're inside the critical section, reset fail counter since we progressed.
if progress && atomic.LoadUint32(&s.d.fsPivotFails) > 1 {
log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails))
atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block
}
// Put unfulfilled tasks back into the retry queue
npeers := s.d.peers.Len()
for hash, task := range req.tasks {
@ -441,12 +432,12 @@ func (s *stateSync) process(req *stateReq) (bool, error) {
// If we've requested the node too many times already, it may be a malicious
// sync where nobody has the right data. Abort.
if len(task.attempts) >= npeers {
return stale, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
}
// Missing item, place into the retry queue.
s.tasks[hash] = task
}
return stale, nil
return nil
}
// processNodeData tries to inject a trie node data blob delivered from a remote

16
eth/handler.go
View file

@ -71,7 +71,6 @@ type ProtocolManager struct {
txpool txPool
blockchain *core.BlockChain
chaindb ethdb.Database
chainconfig *params.ChainConfig
maxPeers int
@ -106,7 +105,6 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
eventMux: mux,
txpool: txpool,
blockchain: blockchain,
chaindb: chaindb,
chainconfig: config,
peers: newPeerSet(),
newPeerCh: make(chan *peer),
@ -257,8 +255,14 @@ func (pm *ProtocolManager) handle(p *peer) error {
p.Log().Debug("Ethereum peer connected", "name", p.Name())
// Execute the Ethereum handshake
td, head, genesis := pm.blockchain.Status()
if err := p.Handshake(pm.networkId, td, head, genesis); err != nil {
var (
genesis = pm.blockchain.Genesis()
head = pm.blockchain.CurrentHeader()
hash = head.Hash()
number = head.Number.Uint64()
td = pm.blockchain.GetTd(hash, number)
)
if err := p.Handshake(pm.networkId, td, hash, genesis.Hash()); err != nil {
p.Log().Debug("Ethereum handshake failed", "err", err)
return err
}
@ -532,7 +536,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Retrieve the requested state entry, stopping if enough was found
if entry, err := pm.chaindb.Get(hash.Bytes()); err == nil {
if entry, err := pm.blockchain.TrieNode(hash); err == nil {
data = append(data, entry)
bytes += len(entry)
}
@ -570,7 +574,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Retrieve the requested block's receipts, skipping if unknown to us
results := core.GetBlockReceipts(pm.chaindb, hash, core.GetBlockNumber(pm.chaindb, hash))
results := pm.blockchain.GetReceiptsByHash(hash)
if results == nil {
if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
continue

16
eth/handler_test.go
View file

@ -56,7 +56,7 @@ func TestProtocolCompatibility(t *testing.T) {
for i, tt := range tests {
ProtocolVersions = []uint{tt.version}
pm, err := newTestProtocolManager(tt.mode, 0, nil, nil)
pm, _, err := newTestProtocolManager(tt.mode, 0, nil, nil)
if pm != nil {
defer pm.Stop()
}
@ -71,7 +71,7 @@ func TestGetBlockHeaders62(t *testing.T) { testGetBlockHeaders(t, 62) }
func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) }
func testGetBlockHeaders(t *testing.T, protocol int) {
pm := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxHashFetch+15, nil, nil)
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -230,7 +230,7 @@ func TestGetBlockBodies62(t *testing.T) { testGetBlockBodies(t, 62) }
func TestGetBlockBodies63(t *testing.T) { testGetBlockBodies(t, 63) }
func testGetBlockBodies(t *testing.T, protocol int) {
pm := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxBlockFetch+15, nil, nil)
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, downloader.MaxBlockFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -337,13 +337,13 @@ func testGetNodeData(t *testing.T, protocol int) {
}
}
// Assemble the test environment
pm := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil)
pm, db := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
// Fetch for now the entire chain db
hashes := []common.Hash{}
for _, key := range pm.chaindb.(*ethdb.MemDatabase).Keys() {
for _, key := range db.Keys() {
if len(key) == len(common.Hash{}) {
hashes = append(hashes, common.BytesToHash(key))
}
@ -429,7 +429,7 @@ func testGetReceipt(t *testing.T, protocol int) {
}
}
// Assemble the test environment
pm := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil)
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -439,7 +439,7 @@ func testGetReceipt(t *testing.T, protocol int) {
block := pm.blockchain.GetBlockByNumber(i)
hashes = append(hashes, block.Hash())
receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash(), block.NumberU64()))
receipts = append(receipts, pm.blockchain.GetReceiptsByHash(block.Hash()))
}
// Send the hash request and verify the response
p2p.Send(peer.app, 0x0f, hashes)
@ -472,7 +472,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool
config = &params.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
gspec = &core.Genesis{Config: config}
genesis = gspec.MustCommit(db)
blockchain, _ = core.NewBlockChain(db, config, pow, vm.Config{})
blockchain, _ = core.NewBlockChain(db, nil, config, pow, vm.Config{})
)
pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db)
if err != nil {

22
eth/helper_test.go
View file

@ -49,7 +49,7 @@ var (
// newTestProtocolManager creates a new protocol manager for testing purposes,
// with the given number of blocks already known, and potential notification
// channels for different events.
func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, error) {
func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, *ethdb.MemDatabase, error) {
var (
evmux = new(event.TypeMux)
engine = ethash.NewFaker()
@ -59,7 +59,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func
Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}},
}
genesis = gspec.MustCommit(db)
blockchain, _ = core.NewBlockChain(db, gspec.Config, engine, vm.Config{})
blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
)
chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator)
if _, err := blockchain.InsertChain(chain); err != nil {
@ -68,22 +68,22 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func
pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db)
if err != nil {
return nil, err
return nil, nil, err
}
pm.Start(1000)
return pm, nil
return pm, db, nil
}
// newTestProtocolManagerMust creates a new protocol manager for testing purposes,
// with the given number of blocks already known, and potential notification
// channels for different events. In case of an error, the constructor force-
// fails the test.
func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) *ProtocolManager {
pm, err := newTestProtocolManager(mode, blocks, generator, newtx)
func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, *ethdb.MemDatabase) {
pm, db, err := newTestProtocolManager(mode, blocks, generator, newtx)
if err != nil {
t.Fatalf("Failed to create protocol manager: %v", err)
}
return pm
return pm, db
}
// testTxPool is a fake, helper transaction pool for testing purposes
@ -166,8 +166,12 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te
tp := &testPeer{app: app, net: net, peer: peer}
// Execute any implicitly requested handshakes and return
if shake {
td, head, genesis := pm.blockchain.Status()
tp.handshake(nil, td, head, genesis)
var (
genesis = pm.blockchain.Genesis()
head = pm.blockchain.CurrentHeader()
td = pm.blockchain.GetTd(head.Hash(), head.Number.Uint64())
)
tp.handshake(nil, td, head.Hash(), genesis.Hash())
}
return tp, errc
}

20
eth/protocol_test.go
View file

@ -41,8 +41,12 @@ func TestStatusMsgErrors62(t *testing.T) { testStatusMsgErrors(t, 62) }
func TestStatusMsgErrors63(t *testing.T) { testStatusMsgErrors(t, 63) }
func testStatusMsgErrors(t *testing.T, protocol int) {
pm := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil)
td, currentBlock, genesis := pm.blockchain.Status()
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil)
var (
genesis = pm.blockchain.Genesis()
head = pm.blockchain.CurrentHeader()
td = pm.blockchain.GetTd(head.Hash(), head.Number.Uint64())
)
defer pm.Stop()
tests := []struct {
@ -55,16 +59,16 @@ func testStatusMsgErrors(t *testing.T, protocol int) {
wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"),
},
{
code: StatusMsg, data: statusData{10, DefaultConfig.NetworkId, td, currentBlock, genesis},
code: StatusMsg, data: statusData{10, DefaultConfig.NetworkId, td, head.Hash(), genesis.Hash()},
wantError: errResp(ErrProtocolVersionMismatch, "10 (!= %d)", protocol),
},
{
code: StatusMsg, data: statusData{uint32(protocol), 999, td, currentBlock, genesis},
code: StatusMsg, data: statusData{uint32(protocol), 999, td, head.Hash(), genesis.Hash()},
wantError: errResp(ErrNetworkIdMismatch, "999 (!= 1)"),
},
{
code: StatusMsg, data: statusData{uint32(protocol), DefaultConfig.NetworkId, td, currentBlock, common.Hash{3}},
wantError: errResp(ErrGenesisBlockMismatch, "0300000000000000 (!= %x)", genesis[:8]),
code: StatusMsg, data: statusData{uint32(protocol), DefaultConfig.NetworkId, td, head.Hash(), common.Hash{3}},
wantError: errResp(ErrGenesisBlockMismatch, "0300000000000000 (!= %x)", genesis.Hash().Bytes()[:8]),
},
}
@ -94,7 +98,7 @@ func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) }
func testRecvTransactions(t *testing.T, protocol int) {
txAdded := make(chan []*types.Transaction)
pm := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, txAdded)
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, txAdded)
pm.acceptTxs = 1 // mark synced to accept transactions
p, _ := newTestPeer("peer", protocol, pm, true)
defer pm.Stop()
@ -121,7 +125,7 @@ func TestSendTransactions62(t *testing.T) { testSendTransactions(t, 62) }
func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) }
func testSendTransactions(t *testing.T, protocol int) {
pm := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil)
pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil)
defer pm.Stop()
// Fill the pool with big transactions.

15
eth/sync.go
View file

@ -189,18 +189,13 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
mode = downloader.FastSync
}
// Run the sync cycle, and disable fast sync if we've went past the pivot block
err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode)
if atomic.LoadUint32(&pm.fastSync) == 1 {
// Disable fast sync if we indeed have something in our chain
if pm.blockchain.CurrentBlock().NumberU64() > 0 {
log.Info("Fast sync complete, auto disabling")
atomic.StoreUint32(&pm.fastSync, 0)
}
}
if err != nil {
if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil {
return
}
if atomic.LoadUint32(&pm.fastSync) == 1 {
log.Info("Fast sync complete, auto disabling")
atomic.StoreUint32(&pm.fastSync, 0)
}
atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done
if head := pm.blockchain.CurrentBlock(); head.NumberU64() > 0 {
// We've completed a sync cycle, notify all peers of new state. This path is

4
eth/sync_test.go
View file

@ -30,12 +30,12 @@ import (
// imported into the blockchain.
func TestFastSyncDisabling(t *testing.T) {
// Create a pristine protocol manager, check that fast sync is left enabled
pmEmpty := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil)
pmEmpty, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil)
if atomic.LoadUint32(&pmEmpty.fastSync) == 0 {
t.Fatalf("fast sync disabled on pristine blockchain")
}
// Create a full protocol manager, check that fast sync gets disabled
pmFull := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil)
pmFull, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil)
if atomic.LoadUint32(&pmFull.fastSync) == 1 {
t.Fatalf("fast sync not disabled on non-empty blockchain")
}

2
eth/tracers/tracer.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify

2
eth/tracers/tracer_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify

9
ethdb/database.go
View file

@ -299,6 +299,11 @@ func (b *ldbBatch) ValueSize() int {
return b.size
}
func (b *ldbBatch) Reset() {
b.b.Reset()
b.size = 0
}
type table struct {
db Database
prefix string
@ -358,3 +363,7 @@ func (tb *tableBatch) Write() error {
func (tb *tableBatch) ValueSize() int {
return tb.batch.ValueSize()
}
func (tb *tableBatch) Reset() {
tb.batch.Reset()
}

2
ethdb/interface.go
View file

@ -41,4 +41,6 @@ type Batch interface {
Putter
ValueSize() int // amount of data in the batch
Write() error
// Reset resets the batch for reuse
Reset()
}

5
ethdb/memory_database.go
View file

@ -123,3 +123,8 @@ func (b *memBatch) Write() error {
func (b *memBatch) ValueSize() int {
return b.size
}
func (b *memBatch) Reset() {
b.writes = b.writes[:0]
b.size = 0
}

2
internal/build/env.go
View file

@ -94,7 +94,7 @@ func LocalEnv() Environment {
}
if env.Branch == "" {
if head != "HEAD" {
env.Branch = strings.TrimLeft(head, "refs/heads/")
env.Branch = strings.TrimPrefix(head, "refs/heads/")
}
}
if info, err := os.Stat(".git/objects"); err == nil && info.IsDir() && env.Tag == "" {

16
internal/cmdtest/test_cmd.go
View file

@ -1,18 +1,18 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package cmdtest

2
internal/ethapi/addrlock.go
View file

@ -1,4 +1,4 @@
// Copyright 2015 The go-ethereum Authors
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify

88
internal/ethapi/api.go
View file

@ -333,28 +333,19 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
return fetchKeystore(s.am).Lock(addr) == nil
}
// SendTransaction will create a transaction from the given arguments and
// tries to sign it with the key associated with args.To. If the given passwd isn't
// able to decrypt the key it fails.
func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) {
// signTransactions sets defaults and signs the given transaction
// NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
// and release it after the transaction has been submitted to the tx pool
func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs, passwd string) (*types.Transaction, error) {
// Look up the wallet containing the requested signer
account := accounts.Account{Address: args.From}
wallet, err := s.am.Find(account)
if err != nil {
return common.Hash{}, err
return nil, err
}
if args.Nonce == nil {
// Hold the addresse's mutex around signing to prevent concurrent assignment of
// the same nonce to multiple accounts.
s.nonceLock.LockAddr(args.From)
defer s.nonceLock.UnlockAddr(args.From)
}
// Set some sanity defaults and terminate on failure
if err := args.setDefaults(ctx, s.b); err != nil {
return common.Hash{}, err
return nil, err
}
// Assemble the transaction and sign with the wallet
tx := args.toTransaction()
@ -363,13 +354,53 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs
if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) {
chainID = config.ChainId
}
signed, err := wallet.SignTxWithPassphrase(account, passwd, tx, chainID)
return wallet.SignTxWithPassphrase(account, passwd, tx, chainID)
}
// SendTransaction will create a transaction from the given arguments and
// tries to sign it with the key associated with args.To. If the given passwd isn't
// able to decrypt the key it fails.
func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) {
if args.Nonce == nil {
// Hold the addresse's mutex around signing to prevent concurrent assignment of
// the same nonce to multiple accounts.
s.nonceLock.LockAddr(args.From)
defer s.nonceLock.UnlockAddr(args.From)
}
signed, err := s.signTransaction(ctx, args, passwd)
if err != nil {
return common.Hash{}, err
}
return submitTransaction(ctx, s.b, signed)
}
// SignTransaction will create a transaction from the given arguments and
// tries to sign it with the key associated with args.To. If the given passwd isn't
// able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast
// to other nodes
func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs, passwd string) (*SignTransactionResult, error) {
// No need to obtain the noncelock mutex, since we won't be sending this
// tx into the transaction pool, but right back to the user
if args.Gas == nil {
return nil, fmt.Errorf("gas not specified")
}
if args.GasPrice == nil {
return nil, fmt.Errorf("gasPrice not specified")
}
if args.Nonce == nil {
return nil, fmt.Errorf("nonce not specified")
}
signed, err := s.signTransaction(ctx, args, passwd)
if err != nil {
return nil, err
}
data, err := rlp.EncodeToBytes(signed)
if err != nil {
return nil, err
}
return &SignTransactionResult{data, signed}, nil
}
// signHash is a helper function that calculates a hash for the given message that can be
// safely used to calculate a signature from.
//
@ -777,7 +808,7 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx
"difficulty": (*hexutil.Big)(head.Difficulty),
"totalDifficulty": (*hexutil.Big)(s.b.GetTd(b.Hash())),
"extraData": hexutil.Bytes(head.Extra),
"size": hexutil.Uint64(uint64(b.Size().Int64())),
"size": hexutil.Uint64(b.Size()),
"gasLimit": hexutil.Uint64(head.GasLimit),
"gasUsed": hexutil.Uint64(head.GasUsed),
"timestamp": (*hexutil.Big)(head.Time),
@ -1104,6 +1135,18 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error {
if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) {
return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`)
}
if args.To == nil {
// Contract creation
var input []byte
if args.Data != nil {
input = *args.Data
} else if args.Input != nil {
input = *args.Input
}
if len(input) == 0 {
return errors.New(`contract creation without any data provided`)
}
}
return nil
}
@ -1221,11 +1264,14 @@ type SignTransactionResult struct {
// The node needs to have the private key of the account corresponding with
// the given from address and it needs to be unlocked.
func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args SendTxArgs) (*SignTransactionResult, error) {
if args.Gas == nil {
return nil, fmt.Errorf("gas not specified")
}
if args.GasPrice == nil {
return nil, fmt.Errorf("gasPrice not specified")
}
if args.Nonce == nil {
// Hold the addresse's mutex around signing to prevent concurrent assignment of
// the same nonce to multiple accounts.
s.nonceLock.LockAddr(args.From)
defer s.nonceLock.UnlockAddr(args.From)
return nil, fmt.Errorf("nonce not specified")
}
if err := args.setDefaults(ctx, s.b); err != nil {
return nil, err

Some files were not shown because too many files have changed in this diff Show more