forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-24 13:46:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge 83595f4967 into 8b865fa9bf

Browse source
This commit is contained in:
Péter Szilágyi 2015-10-02 11:18:25 +00:00
commit 581284f099
37 changed files with 2838 additions and 1398 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/geth/main.go
View file

@ -288,6 +288,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
utils.DataDirFlag,
utils.BlockchainVersionFlag,
utils.OlympicFlag,
utils.EthModeFlag,
utils.EthVersionFlag,
utils.CacheFlag,
utils.JSpathFlag,

24
cmd/utils/flags.go
View file

@ -27,6 +27,7 @@ import (
"path/filepath"
"runtime"
"strconv"
"strings"
"github.com/codegangsta/cli"
"github.com/ethereum/ethash"
@ -142,9 +143,14 @@ var (
Name: "olympic",
Usage: "Use olympic style protocol",
}
EthModeFlag = cli.StringFlag{
Name: "mode",
Value: "archive",
Usage: "Client mode of operation (archive, full, light)",
}
EthVersionFlag = cli.IntFlag{
Name: "eth",
Value: 62,
Value: 63,
Usage: "Highest eth protocol to advertise (temporary, dev option)",
}
@ -413,12 +419,25 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config {
if err != nil {
glog.V(logger.Error).Infoln("WARNING: No etherbase set and no accounts found as default")
}
// Resolve the mode of opeation from the string flag
var clientMode eth.Mode
switch strings.ToLower(ctx.GlobalString(EthModeFlag.Name)) {
case "archive":
clientMode = eth.ArchiveMode
case "full":
clientMode = eth.FullMode
case "light":
clientMode = eth.LightMode
default:
glog.Fatalf("Unknown node type requested: %s", ctx.GlobalString(EthModeFlag.Name))
}
// Assemble the entire eth configuration and return
cfg := &eth.Config{
Name: common.MakeName(clientID, version),
DataDir: MustDataDir(ctx),
GenesisNonce: ctx.GlobalInt(GenesisNonceFlag.Name),
GenesisFile: ctx.GlobalString(GenesisFileFlag.Name),
Mode: clientMode,
BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name),
DatabaseCache: ctx.GlobalInt(CacheFlag.Name),
SkipBcVersionCheck: false,
@ -476,7 +495,6 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config {
glog.V(logger.Info).Infoln("dev mode enabled")
}
return cfg
}

2
core/bench_test.go
View file

@ -163,7 +163,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
// Generate a chain of b.N blocks using the supplied block
// generator function.
genesis := WriteGenesisBlockForTesting(db, GenesisAccount{benchRootAddr, benchRootFunds})
chain := GenerateChain(genesis, db, b.N, gen)
chain, _ := GenerateChain(genesis, db, b.N, gen)
// Time the insertion of the new chain.
// State and blocks are stored in the same DB.

19
core/block_processor.go
View file

@ -110,7 +110,7 @@ func (self *BlockProcessor) ApplyTransaction(gp GasPool, statedb *state.StateDB,
}
logs := statedb.GetLogs(tx.Hash())
receipt.SetLogs(logs)
receipt.Logs = logs
receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
glog.V(logger.Debug).Infoln(receipt)
@ -360,18 +360,31 @@ func (sm *BlockProcessor) GetLogs(block *types.Block) (logs state.Logs, err erro
receipts := GetBlockReceipts(sm.chainDb, block.Hash())
// coalesce logs
for _, receipt := range receipts {
logs = append(logs, receipt.Logs()...)
logs = append(logs, receipt.Logs...)
}
return logs, nil
}
// ValidateHeader verifies the validity of a header, relying on the database and
// POW behind the block processor.
func (sm *BlockProcessor) ValidateHeader(header *types.Header, checkPow, uncle bool) error {
// Short circuit if the header's already known or its parent missing
if sm.bc.HasHeader(header.Hash()) {
return nil
}
if parent := sm.bc.GetHeader(header.ParentHash); parent == nil {
return ParentError(header.ParentHash)
} else {
return ValidateHeader(sm.Pow, header, parent, checkPow, uncle)
}
}
// See YP section 4.3.4. "Block Header Validity"
// Validates a header. Returns an error if the header is invalid.
func ValidateHeader(pow pow.PoW, header *types.Header, parent *types.Header, checkPow, uncle bool) error {
if big.NewInt(int64(len(header.Extra))).Cmp(params.MaximumExtraDataSize) == 1 {
return fmt.Errorf("Header extra data too long (%d)", len(header.Extra))
}
if uncle {
if header.Time.Cmp(common.MaxBig) == 1 {
return BlockTSTooBigErr

20
core/block_processor_test.go
View file

@ -69,16 +69,16 @@ func TestPutReceipt(t *testing.T) {
hash[0] = 2
receipt := new(types.Receipt)
receipt.SetLogs(state.Logs{&state.Log{
Address: addr,
Topics: []common.Hash{hash},
Data: []byte("hi"),
Number: 42,
TxHash: hash,
TxIndex: 0,
BlockHash: hash,
Index: 0,
}})
receipt.Logs = state.Logs{&state.Log{
Address: addr,
Topics: []common.Hash{hash},
Data: []byte("hi"),
BlockNumber: 42,
TxHash: hash,
TxIndex: 0,
BlockHash: hash,
Index: 0,
}}
PutReceipts(db, types.Receipts{receipt})
receipt = GetReceipt(db, common.Hash{})

66
core/chain_makers.go
View file

@ -99,7 +99,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
b.header.GasUsed.Add(b.header.GasUsed, gas)
receipt := types.NewReceipt(root.Bytes(), b.header.GasUsed)
logs := b.statedb.GetLogs(tx.Hash())
receipt.SetLogs(logs)
receipt.Logs = logs
receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
b.txs = append(b.txs, tx)
b.receipts = append(b.receipts, receipt)
@ -155,10 +155,10 @@ func (b *BlockGen) OffsetTime(seconds int64) {
// Blocks created by GenerateChain do not contain valid proof of work
// values. Inserting them into ChainManager requires use of FakePow or
// a similar non-validating proof of work implementation.
func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) []*types.Block {
func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) {
statedb := state.New(parent.Root(), db)
blocks := make(types.Blocks, n)
genblock := func(i int, h *types.Header) *types.Block {
blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
genblock := func(i int, h *types.Header) (*types.Block, types.Receipts) {
b := &BlockGen{parent: parent, i: i, chain: blocks, header: h, statedb: statedb}
if gen != nil {
gen(i, b)
@ -169,15 +169,16 @@ func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int,
panic(fmt.Sprintf("state write error: %v", err))
}
h.Root = root
return types.NewBlock(h, b.txs, b.uncles, b.receipts)
return types.NewBlock(h, b.txs, b.uncles, b.receipts), b.receipts
}
for i := 0; i < n; i++ {
header := makeHeader(parent, statedb)
block := genblock(i, header)
block, receipt := genblock(i, header)
blocks[i] = block
receipts[i] = receipt
parent = block
}
return blocks
return blocks, receipts
}
func makeHeader(parent *types.Block, state *state.StateDB) *types.Header {
@ -199,26 +200,51 @@ func makeHeader(parent *types.Block, state *state.StateDB) *types.Header {
}
}
// newCanonical creates a new deterministic canonical chain by running
// InsertChain on the result of makeChain.
func newCanonical(n int, db ethdb.Database) (*BlockProcessor, error) {
// newCanonical creates a chain database, and injects a deterministic canonical
// chain. Depending on the full flag, if creates either a full block chain or a
// header only chain.
func newCanonical(n int, full bool) (ethdb.Database, *BlockProcessor, error) {
// Create te new chain database
db, _ := ethdb.NewMemDatabase()
evmux := &event.TypeMux{}
WriteTestNetGenesisBlock(db, 0)
// Initialize a fresh chain with only a genesis block
genesis, _ := WriteTestNetGenesisBlock(db, 0)
chainman, _ := NewChainManager(db, FakePow{}, evmux)
bman := NewBlockProcessor(db, FakePow{}, chainman, evmux)
bman.bc.SetProcessor(bman)
parent := bman.bc.CurrentBlock()
processor := NewBlockProcessor(db, FakePow{}, chainman, evmux)
processor.bc.SetProcessor(processor)
// Create and inject the requested chain
if n == 0 {
return bman, nil
return db, processor, nil
}
lchain := makeChain(parent, n, db, canonicalSeed)
_, err := bman.bc.InsertChain(lchain)
return bman, err
if full {
// Full block-chain requested
blocks := makeBlockChain(genesis, n, db, canonicalSeed)
_, err := chainman.InsertChain(blocks)
return db, processor, err
}
// Header-only chain requested
headers := makeHeaderChain(genesis.Header(), n, db, canonicalSeed)
_, err := chainman.InsertHeaderChain(headers, true)
return db, processor, err
}
func makeChain(parent *types.Block, n int, db ethdb.Database, seed int) []*types.Block {
return GenerateChain(parent, db, n, func(i int, b *BlockGen) {
// makeHeaderChain creates a deterministic chain of headers rooted at parent.
func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) []*types.Header {
blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, db, seed)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
return headers
}
// makeBlockChain creates a deterministic chain of blocks rooted at parent.
func makeBlockChain(parent *types.Block, n int, db ethdb.Database, seed int) []*types.Block {
blocks, _ := GenerateChain(parent, db, n, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)})
})
return blocks
}

2
core/chain_makers_test.go
View file

@ -47,7 +47,7 @@ func ExampleGenerateChain() {
// This call generates a chain of 5 blocks. The function runs for
// each block and adds different features to gen based on the
// block index.
chain := GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
chain, _ := GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
// In block 1, addr1 sends addr2 some ether.

505
core/chain_manager.go
View file

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/logger"
@ -69,10 +70,10 @@ type ChainManager struct {
chainmu sync.RWMutex
tsmu sync.RWMutex
checkpoint int // checkpoint counts towards the new checkpoint
td *big.Int
currentBlock *types.Block
currentGasLimit *big.Int
checkpoint int // checkpoint counts towards the new checkpoint
currentHeader *types.Header // Current head of the header chain (may be above the block chain!)
currentBlock *types.Block // Current head of the block chain
currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
headerCache *lru.Cache // Cache for the most recent block headers
bodyCache *lru.Cache // Cache for the most recent block bodies
@ -123,20 +124,15 @@ func NewChainManager(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*
}
glog.V(logger.Info).Infoln("WARNING: Wrote default ethereum genesis block")
}
if err := bc.setLastState(); err != nil {
if err := bc.loadLastState(); err != nil {
return nil, err
}
// Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain
for hash, _ := range BadHashes {
if block := bc.GetBlock(hash); block != nil {
glog.V(logger.Error).Infof("Found bad hash. Reorganising chain to state %x\n", block.ParentHash().Bytes()[:4])
block = bc.GetBlock(block.ParentHash())
if block == nil {
glog.Fatal("Unable to complete. Parent block not found. Corrupted DB?")
}
bc.SetHead(block)
glog.V(logger.Error).Infoln("Chain reorg was successfull. Resuming normal operation")
if header := bc.GetHeader(hash); header != nil {
glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4])
bc.SetHead(header.Number.Uint64() - 1)
glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation")
}
}
// Take ownership of this particular state
@ -144,30 +140,117 @@ func NewChainManager(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*
return bc, nil
}
func (bc *ChainManager) SetHead(head *types.Block) {
// loadLastState loads the last known chain state from the database. This method
// assumes that the chain manager mutex is held.
func (self *ChainManager) loadLastState() error {
// Restore the last known head block
head := GetHeadBlockHash(self.chainDb)
if head == (common.Hash{}) {
// Corrupt or empty database, init from scratch
self.Reset()
} else {
if block := self.GetBlock(head); block != nil {
// Block found, set as the current head
self.currentBlock = block
} else {
// Database corrupted, try to restore it
glog.Infof("last block (%x…) not found, recovering...", head[:4])
if self.recover() {
glog.Infof("recover successful")
} else {
glog.Fatalf("recover failed, please report")
}
}
}
// Restore the last known head header
self.currentHeader = self.currentBlock.Header()
if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) {
if header := self.GetHeader(head); header != nil {
self.currentHeader = header
}
}
// Restore the last known head fast block
self.currentFastBlock = self.currentBlock
if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) {
if block := self.GetBlock(head); block != nil {
self.currentFastBlock = block
}
}
// Issue a status log and return
headerTd := self.GetTd(self.currentHeader.Hash())
blockTd := self.GetTd(self.currentBlock.Hash())
fastTd := self.GetTd(self.currentFastBlock.Hash())
glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", self.currentHeader.Number, self.currentHeader.Hash().Bytes()[:4], headerTd)
glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd)
glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd)
return nil
}
// SetHead rewind the local chain to a new head entity. In the case of headers,
// everything above the new head will be deleted and the new one set. In the case
// of blocks though, the head may be further rewound if block bodies are missing
// (non-archive nodes after a fast sync).
func (bc *ChainManager) SetHead(head uint64) {
bc.mu.Lock()
defer bc.mu.Unlock()
for block := bc.currentBlock; block != nil && block.Hash() != head.Hash(); block = bc.GetBlock(block.ParentHash()) {
DeleteBlock(bc.chainDb, block.Hash())
// Figure out the highest known canonical assignment
height := uint64(0)
if bc.currentHeader != nil {
if hh := bc.currentHeader.Number.Uint64(); hh > height {
height = hh
}
}
if bc.currentBlock != nil {
if bh := bc.currentBlock.NumberU64(); bh > height {
height = bh
}
}
if bc.currentFastBlock != nil {
if fbh := bc.currentFastBlock.NumberU64(); fbh > height {
height = fbh
}
}
// Gather all the hashes that need deletion
drop := make(map[common.Hash]struct{})
for bc.currentHeader != nil && bc.currentHeader.Number.Uint64() > head {
drop[bc.currentHeader.Hash()] = struct{}{}
bc.currentHeader = bc.GetHeader(bc.currentHeader.ParentHash)
}
for bc.currentBlock != nil && bc.currentBlock.NumberU64() > head {
drop[bc.currentBlock.Hash()] = struct{}{}
bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash())
}
for bc.currentFastBlock != nil && bc.currentFastBlock.NumberU64() > head {
drop[bc.currentFastBlock.Hash()] = struct{}{}
bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash())
}
// Roll back the canonical chain numbering
for i := height; i > head; i-- {
DeleteCanonicalHash(bc.chainDb, i)
}
// Delete everything found by the above rewind
for hash, _ := range drop {
DeleteHeader(bc.chainDb, hash)
DeleteBody(bc.chainDb, hash)
DeleteTd(bc.chainDb, hash)
}
// Clear out any stale content from the caches
bc.headerCache.Purge()
bc.bodyCache.Purge()
bc.bodyRLPCache.Purge()
bc.blockCache.Purge()
bc.futureBlocks.Purge()
bc.currentBlock = head
bc.setTotalDifficulty(bc.GetTd(head.Hash()))
bc.insert(head)
bc.setLastState()
}
func (self *ChainManager) Td() *big.Int {
self.mu.RLock()
defer self.mu.RUnlock()
return new(big.Int).Set(self.td)
// Update all computed fields to the new head
if bc.currentBlock == nil {
bc.currentBlock = bc.genesisBlock
}
bc.insert(bc.currentBlock)
bc.loadLastState()
}
func (self *ChainManager) GasLimit() *big.Int {
@ -184,6 +267,17 @@ func (self *ChainManager) LastBlockHash() common.Hash {
return self.currentBlock.Hash()
}
// CurrentHeader retrieves the current head header of the canonical chain. The
// header is retrieved from the chain manager's internal cache.
func (self *ChainManager) CurrentHeader() *types.Header {
self.mu.RLock()
defer self.mu.RUnlock()
return self.currentHeader
}
// CurrentBlock retrieves the current head block of the canonical chain. The
// block is retrieved from the chain manager's internal cache.
func (self *ChainManager) CurrentBlock() *types.Block {
self.mu.RLock()
defer self.mu.RUnlock()
@ -191,11 +285,20 @@ func (self *ChainManager) CurrentBlock() *types.Block {
return self.currentBlock
}
// CurrentFastBlock retrieves the current fast-sync head block of the canonical
// chain. The block is retrieved from the chain manager's internal cache.
func (self *ChainManager) CurrentFastBlock() *types.Block {
self.mu.RLock()
defer self.mu.RUnlock()
return self.currentFastBlock
}
func (self *ChainManager) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) {
self.mu.RLock()
defer self.mu.RUnlock()
return new(big.Int).Set(self.td), self.currentBlock.Hash(), self.genesisBlock.Hash()
return self.GetTd(self.currentBlock.Hash()), self.currentBlock.Hash(), self.genesisBlock.Hash()
}
func (self *ChainManager) SetProcessor(proc types.BlockProcessor) {
@ -206,6 +309,8 @@ func (self *ChainManager) State() *state.StateDB {
return state.New(self.CurrentBlock().Root(), self.chainDb)
}
// recover tries to retrieve a secondary last known good state from the database
// na drestore that as the chain head.
func (bc *ChainManager) recover() bool {
data, _ := bc.chainDb.Get([]byte("checkpoint"))
if len(data) != 0 {
@ -215,42 +320,23 @@ func (bc *ChainManager) recover() bool {
glog.Fatalf("failed to write database head number: %v", err)
}
if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to write database head hash: %v", err)
glog.Fatalf("failed to write database head block hash: %v", err)
}
if err := WriteHeadHeaderHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to write database head header hash: %v", err)
}
if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to write database head fast block hash: %v", err)
}
bc.currentBlock = block
bc.currentHeader = block.Header()
bc.currentFastBlock = block
return true
}
}
return false
}
func (bc *ChainManager) setLastState() error {
head := GetHeadBlockHash(bc.chainDb)
if head != (common.Hash{}) {
block := bc.GetBlock(head)
if block != nil {
bc.currentBlock = block
} else {
glog.Infof("LastBlock (%x) not found. Recovering...\n", head)
if bc.recover() {
glog.Infof("Recover successful")
} else {
glog.Fatalf("Recover failed. Please report")
}
}
} else {
bc.Reset()
}
bc.td = bc.GetTd(bc.currentBlock.Hash())
bc.currentGasLimit = CalcGasLimit(bc.currentBlock)
if glog.V(logger.Info) {
glog.Infof("Last block (#%v) %x TD=%v\n", bc.currentBlock.Number(), bc.currentBlock.Hash(), bc.td)
}
return nil
}
// Reset purges the entire blockchain, restoring it to its genesis state.
func (bc *ChainManager) Reset() {
bc.ResetWithGenesisBlock(bc.genesisBlock)
@ -259,19 +345,12 @@ func (bc *ChainManager) Reset() {
// ResetWithGenesisBlock purges the entire blockchain, restoring it to the
// specified genesis state.
func (bc *ChainManager) ResetWithGenesisBlock(genesis *types.Block) {
// Dump the entire block chain and purge the caches
bc.SetHead(0)
bc.mu.Lock()
defer bc.mu.Unlock()
// Dump the entire block chain and purge the caches
for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.ParentHash()) {
DeleteBlock(bc.chainDb, block.Hash())
}
bc.headerCache.Purge()
bc.bodyCache.Purge()
bc.bodyRLPCache.Purge()
bc.blockCache.Purge()
bc.futureBlocks.Purge()
// Prepare the genesis block and reinitialize the chain
if err := WriteTd(bc.chainDb, genesis.Hash(), genesis.Difficulty()); err != nil {
glog.Fatalf("failed to write genesis block TD: %v", err)
@ -282,7 +361,8 @@ func (bc *ChainManager) ResetWithGenesisBlock(genesis *types.Block) {
bc.genesisBlock = genesis
bc.insert(bc.genesisBlock)
bc.currentBlock = bc.genesisBlock
bc.setTotalDifficulty(genesis.Difficulty())
bc.currentHeader = bc.genesisBlock.Header()
bc.currentFastBlock = bc.genesisBlock
}
// Export writes the active chain to the given writer.
@ -318,15 +398,25 @@ func (self *ChainManager) ExportN(w io.Writer, first uint64, last uint64) error
return nil
}
// insert injects a block into the current chain block chain. Note, this function
// assumes that the `mu` mutex is held!
// insert injects a new head block into the current block chain. This method
// assumes that the block is indeed a true head. It will also reset the head
// header and the head fast sync block to this very same block to prevent them
// from diverging on a different header chain.
//
// Note, this function assumes that the `mu` mutex is held!
func (bc *ChainManager) insert(block *types.Block) {
// Add the block to the canonical chain number scheme and mark as the head
if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil {
glog.Fatalf("failed to insert block number: %v", err)
}
if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to insert block number: %v", err)
glog.Fatalf("failed to insert head block hash: %v", err)
}
if err := WriteHeadHeaderHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to insert head header hash: %v", err)
}
if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to insert head fast block hash: %v", err)
}
// Add a new restore point if we reached some limit
bc.checkpoint++
@ -336,8 +426,10 @@ func (bc *ChainManager) insert(block *types.Block) {
}
bc.checkpoint = 0
}
// Update the internal internal state with the head block
// Update the internal state with the head block
bc.currentBlock = block
bc.currentHeader = block.Header()
bc.currentFastBlock = block
}
// Accessors
@ -493,19 +585,15 @@ func (self *ChainManager) GetBlocksFromHash(hash common.Hash, n int) (blocks []*
return
}
func (self *ChainManager) GetUnclesInChain(block *types.Block, length int) (uncles []*types.Header) {
// GetUnclesInChain retrieves all the uncles from a given block backwards until
// a specific distance is reached.
func (self *ChainManager) GetUnclesInChain(block *types.Block, length int) []*types.Header {
uncles := []*types.Header{}
for i := 0; block != nil && i < length; i++ {
uncles = append(uncles, block.Uncles()...)
block = self.GetBlock(block.ParentHash())
}
return
}
// setTotalDifficulty updates the TD of the chain manager. Note, this function
// assumes that the `mu` mutex is held!
func (bc *ChainManager) setTotalDifficulty(td *big.Int) {
bc.td = new(big.Int).Set(td)
return uncles
}
func (bc *ChainManager) Stop() {
@ -548,6 +636,224 @@ const (
SideStatTy
)
// writeHeader writes a header into the local chain, given that its parent is
// already known. If the total difficulty of the newly inserted header becomes
// greater than the old known TD, the canonical chain is re-routed.
//
// Note: This method is not concurrent-safe with inserting blocks simultaneously
// into the chain, as side effects caused by reorganizations cannot be emulated
// without the real blocks. Hence, writing headers directly should only be done
// in two scenarios: pure-header mode of operation (light clients), or properly
// separated header/block phases (non-archive clients).
func (self *ChainManager) writeHeader(header *types.Header) error {
self.wg.Add(1)
defer self.wg.Done()
// Calculate the total difficulty of the header
ptd := self.GetTd(header.ParentHash)
if ptd == nil {
return ParentError(header.ParentHash)
}
td := new(big.Int).Add(header.Difficulty, ptd)
// Make sure no inconsistent state is leaked during insertion
self.mu.Lock()
defer self.mu.Unlock()
// If the total difficulty is higher than our known, add it to the canonical chain
if td.Cmp(self.GetTd(self.currentHeader.Hash())) > 0 {
// Delete any canonical number assignments above the new head
for i := header.Number.Uint64() + 1; GetCanonicalHash(self.chainDb, i) != (common.Hash{}); i++ {
DeleteCanonicalHash(self.chainDb, i)
}
// Overwrite any stale canonical number assignments
head := self.GetHeader(header.ParentHash)
for GetCanonicalHash(self.chainDb, head.Number.Uint64()) != head.Hash() {
WriteCanonicalHash(self.chainDb, head.Hash(), head.Number.Uint64())
head = self.GetHeader(head.ParentHash)
}
// Extend the canonical chain with the new header
if err := WriteCanonicalHash(self.chainDb, header.Hash(), header.Number.Uint64()); err != nil {
glog.Fatalf("failed to insert header number: %v", err)
}
if err := WriteHeadHeaderHash(self.chainDb, header.Hash()); err != nil {
glog.Fatalf("failed to insert head header hash: %v", err)
}
self.currentHeader = types.CopyHeader(header)
}
// Irrelevant of the canonical status, write the header itself to the database
if err := WriteTd(self.chainDb, header.Hash(), td); err != nil {
glog.Fatalf("failed to write header total difficulty: %v", err)
}
if err := WriteHeader(self.chainDb, header); err != nil {
glog.Fatalf("filed to write header contents: %v", err)
}
return nil
}
// InsertHeaderChain will attempt to insert the given header chain in to the
// local chain, possibly creating a fork. If an error is returned, it will
// return the index number of the failing header as well an error describing
// what went wrong.
//
// The verify parameter can be used to fine tune whether nonce verification
// should be done or not. The reason behind the optional check is because some
// of the header retrieval mechanisms already need to verfy nonces, as well as
// because nonces can be verified sparsely, not needing to check each.
func (self *ChainManager) InsertHeaderChain(chain []*types.Header, verify bool) (int, error) {
self.wg.Add(1)
defer self.wg.Done()
// Make sure only one thread manipulates the chain at once
self.chainmu.Lock()
defer self.chainmu.Unlock()
// Collect some import statistics to report on
stats := struct{ processed, ignored int }{}
start := time.Now()
// Start the parallel nonce verifier, with a fake nonce if not requested
verifier := self.pow
if !verify {
verifier = FakePow{}
}
nonceAbort, nonceResults := verifyNoncesFromHeaders(verifier, chain)
defer close(nonceAbort)
// Iterate over the headers, inserting any new ones
complete := make([]bool, len(chain))
for i, header := range chain {
// Short circuit insertion if shutting down
if atomic.LoadInt32(&self.procInterrupt) == 1 {
glog.V(logger.Debug).Infoln("premature abort during header chain processing")
break
}
hash := header.Hash()
// Accumulate verification results until the next header is verified
for !complete[i] {
if res := <-nonceResults; res.valid {
complete[res.index] = true
} else {
header := chain[res.index]
return res.index, &BlockNonceErr{
Hash: header.Hash(),
Number: new(big.Int).Set(header.Number),
Nonce: header.Nonce.Uint64(),
}
}
}
if BadHashes[hash] {
glog.V(logger.Error).Infof("bad header %d [%x…], known bad hash", header.Number, hash)
return i, BadHashError(hash)
}
// Write the header to the chain and get the status
if self.HasHeader(hash) {
stats.ignored++
continue
}
if err := self.writeHeader(header); err != nil {
return i, err
}
stats.processed++
}
// Report some public statistics so the user has a clue what's going on
first, last := chain[0], chain[len(chain)-1]
glog.V(logger.Info).Infof("imported %d header(s) (%d ignored) in %v. #%v [%x… / %x…]", stats.processed, stats.ignored,
time.Since(start), last.Number, first.Hash().Bytes()[:4], last.Hash().Bytes()[:4])
return 0, nil
}
// InsertReceiptChain attempts to complete an already existing header chain with
// transaction and receipt data.
func (self *ChainManager) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
self.wg.Add(1)
defer self.wg.Done()
// Make sure only one thread manipulates the chain at once
self.chainmu.Lock()
defer self.chainmu.Unlock()
// Collect some import statistics to report on
stats := struct{ processed, ignored int }{}
start := time.Now()
// Iterate over the blocks and receipts, inserting any new ones
for i := 0; i < len(blockChain) && i < len(receiptChain); i++ {
block, receipts := blockChain[i], receiptChain[i]
// Short circuit insertion if shutting down
if atomic.LoadInt32(&self.procInterrupt) == 1 {
glog.V(logger.Debug).Infoln("premature abort during receipt chain processing")
break
}
// Short circuit if the owner header is unknown
if !self.HasHeader(block.Hash()) {
glog.V(logger.Debug).Infof("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
}
// Skip if the entire data is already known
if self.HasBlock(block.Hash()) {
stats.ignored++
continue
}
// Compute all the non-consensus fields of the receipts
transactions, logIndex := block.Transactions(), uint(0)
for j := 0; j < len(receipts); j++ {
// The transaction hash can be retrieved from the transaction itself
receipts[j].TxHash = transactions[j].Hash()
// The contract address can be derived from the transaction itself
if MessageCreatesContract(transactions[j]) {
from, _ := transactions[j].From()
receipts[j].ContractAddress = crypto.CreateAddress(from, transactions[j].Nonce())
}
// The used gas can be calculated based on previous receipts
if j == 0 {
receipts[j].GasUsed = new(big.Int).Set(receipts[j].CumulativeGasUsed)
} else {
receipts[j].GasUsed = new(big.Int).Sub(receipts[j].CumulativeGasUsed, receipts[j-1].CumulativeGasUsed)
}
// The derived log fields can simply be set from the block and transaction
for k := 0; k < len(receipts[j].Logs); k++ {
receipts[j].Logs[k].BlockNumber = block.NumberU64()
receipts[j].Logs[k].BlockHash = block.Hash()
receipts[j].Logs[k].TxHash = receipts[j].TxHash
receipts[j].Logs[k].TxIndex = uint(j)
receipts[j].Logs[k].Index = logIndex
logIndex++
}
}
// Write all the data out into the database
if err := WriteBody(self.chainDb, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil {
glog.Fatalf("failed to write block body: %v", err)
return i, err
}
if err := PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
glog.Fatalf("failed to write block receipts: %v", err)
return i, err
}
// Update the head fast sync block if better
self.mu.Lock()
if self.GetTd(self.currentFastBlock.Hash()).Cmp(self.GetTd(block.Hash())) < 0 {
if err := WriteHeadFastBlockHash(self.chainDb, block.Hash()); err != nil {
glog.Fatalf("failed to update head fast block hash: %v", err)
}
self.currentFastBlock = block
}
self.mu.Unlock()
stats.processed++
}
// Report some public statistics so the user has a clue what's going on
first, last := blockChain[0], blockChain[len(blockChain)-1]
glog.V(logger.Info).Infof("imported %d receipt(s) (%d ignored) in %v. #%d [%x… / %x…]", stats.processed, stats.ignored,
time.Since(start), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4])
return 0, nil
}
// WriteBlock writes the block to the chain.
func (self *ChainManager) WriteBlock(block *types.Block) (status writeStatus, err error) {
self.wg.Add(1)
@ -560,38 +866,31 @@ func (self *ChainManager) WriteBlock(block *types.Block) (status writeStatus, er
}
td := new(big.Int).Add(block.Difficulty(), ptd)
self.mu.RLock()
cblock := self.currentBlock
self.mu.RUnlock()
// Make sure no inconsistent state is leaked during insertion
self.mu.Lock()
defer self.mu.Unlock()
// Compare the TD of the last known block in the canonical chain to make sure it's greater.
// At this point it's possible that a different chain (fork) becomes the new canonical chain.
if td.Cmp(self.Td()) > 0 {
// chain fork
if block.ParentHash() != cblock.Hash() {
// during split we merge two different chains and create the new canonical chain
err := self.reorg(cblock, block)
if err != nil {
// If the total difficulty is higher than our known, add it to the canonical chain
if td.Cmp(self.GetTd(self.currentBlock.Hash())) > 0 {
// Reorganize the chain if the parent is not the head block
if block.ParentHash() != self.currentBlock.Hash() {
if err := self.reorg(self.currentBlock, block); err != nil {
return NonStatTy, err
}
}
status = CanonStatTy
self.mu.Lock()
self.setTotalDifficulty(td)
// Insert the block as the new head of the chain
self.insert(block)
self.mu.Unlock()
status = CanonStatTy
} else {
status = SideStatTy
}
// Irrelevant of the canonical status, write the block itself to the database
if err := WriteTd(self.chainDb, block.Hash(), td); err != nil {
glog.Fatalf("failed to write block total difficulty: %v", err)
}
if err := WriteBlock(self.chainDb, block); err != nil {
glog.Fatalf("filed to write block contents: %v", err)
}
// Delete from future blocks
self.futureBlocks.Remove(block.Hash())
return
@ -625,7 +924,7 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) {
txcount := 0
for i, block := range chain {
if atomic.LoadInt32(&self.procInterrupt) == 1 {
glog.V(logger.Debug).Infoln("Premature abort during chain processing")
glog.V(logger.Debug).Infoln("Premature abort during block chain processing")
break
}
@ -681,7 +980,7 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) {
return i, err
}
if err := PutBlockReceipts(self.chainDb, block, receipts); err != nil {
if err := PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
glog.V(logger.Warn).Infoln("error writing block receipts:", err)
}
@ -731,9 +1030,6 @@ func (self *ChainManager) InsertChain(chain types.Blocks) (int, error) {
// to be part of the new canonical chain and accumulates potential missing transactions and post an
// event about them
func (self *ChainManager) reorg(oldBlock, newBlock *types.Block) error {
self.mu.Lock()
defer self.mu.Unlock()
var (
newChain types.Blocks
commonBlock *types.Block
@ -826,7 +1122,6 @@ out:
// We need some control over the mining operation. Acquiring locks and waiting for the miner to create new block takes too long
// and in most cases isn't even necessary.
if self.currentBlock.Hash() == event.Hash {
self.currentGasLimit = CalcGasLimit(event.Block)
self.eventMux.Post(ChainHeadEvent{event.Block})
}
}

743
core/chain_manager_test.go
View file

@ -64,44 +64,58 @@ func theChainManager(db ethdb.Database, t *testing.T) *ChainManager {
}
// Test fork of length N starting from block i
func testFork(t *testing.T, bman *BlockProcessor, i, N int, f func(td1, td2 *big.Int)) {
// switch databases to process the new chain
db, err := ethdb.NewMemDatabase()
if err != nil {
t.Fatal("Failed to create db:", err)
}
// copy old chain up to i into new db with deterministic canonical
bman2, err := newCanonical(i, db)
func testFork(t *testing.T, processor *BlockProcessor, i, n int, full bool, comparator func(td1, td2 *big.Int)) {
// Copy old chain up to #i into a new db
db, processor2, err := newCanonical(i, full)
if err != nil {
t.Fatal("could not make new canonical in testFork", err)
}
// assert the bmans have the same block at i
bi1 := bman.bc.GetBlockByNumber(uint64(i)).Hash()
bi2 := bman2.bc.GetBlockByNumber(uint64(i)).Hash()
if bi1 != bi2 {
fmt.Printf("%+v\n%+v\n\n", bi1, bi2)
t.Fatal("chains do not have the same hash at height", i)
// Assert the chains have the same header/block at #i
var hash1, hash2 common.Hash
if full {
hash1 = processor.bc.GetBlockByNumber(uint64(i)).Hash()
hash2 = processor2.bc.GetBlockByNumber(uint64(i)).Hash()
} else {
hash1 = processor.bc.GetHeaderByNumber(uint64(i)).Hash()
hash2 = processor2.bc.GetHeaderByNumber(uint64(i)).Hash()
}
bman2.bc.SetProcessor(bman2)
// extend the fork
parent := bman2.bc.CurrentBlock()
chainB := makeChain(parent, N, db, forkSeed)
_, err = bman2.bc.InsertChain(chainB)
if err != nil {
t.Fatal("Insert chain error for fork:", err)
if hash1 != hash2 {
t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", i, hash2, hash1)
}
tdpre := bman.bc.Td()
// Test the fork's blocks on the original chain
td, err := testChain(chainB, bman)
if err != nil {
t.Fatal("expected chainB not to give errors:", err)
// Extend the newly created chain
var (
blockChainB []*types.Block
headerChainB []*types.Header
)
if full {
blockChainB = makeBlockChain(processor2.bc.CurrentBlock(), n, db, forkSeed)
if _, err := processor2.bc.InsertChain(blockChainB); err != nil {
t.Fatalf("failed to insert forking chain: %v", err)
}
} else {
headerChainB = makeHeaderChain(processor2.bc.CurrentHeader(), n, db, forkSeed)
if _, err := processor2.bc.InsertHeaderChain(headerChainB, true); err != nil {
t.Fatalf("failed to insert forking chain: %v", err)
}
}
// Compare difficulties
f(tdpre, td)
// Sanity check that the forked chain can be imported into the original
var tdPre, tdPost *big.Int
// Loop over parents making sure reconstruction is done properly
if full {
tdPre = processor.bc.GetTd(processor.bc.CurrentBlock().Hash())
if err := testBlockChainImport(blockChainB, processor); err != nil {
t.Fatalf("failed to import forked block chain: %v", err)
}
tdPost = processor.bc.GetTd(blockChainB[len(blockChainB)-1].Hash())
} else {
tdPre = processor.bc.GetTd(processor.bc.CurrentHeader().Hash())
if err := testHeaderChainImport(headerChainB, processor); err != nil {
t.Fatalf("failed to import forked header chain: %v", err)
}
tdPost = processor.bc.GetTd(headerChainB[len(headerChainB)-1].Hash())
}
// Compare the total difficulties of the chains
comparator(tdPre, tdPost)
}
func printChain(bc *ChainManager) {
@ -111,22 +125,41 @@ func printChain(bc *ChainManager) {
}
}
// process blocks against a chain
func testChain(chainB types.Blocks, bman *BlockProcessor) (*big.Int, error) {
for _, block := range chainB {
_, _, err := bman.bc.processor.Process(block)
if err != nil {
// testBlockChainImport tries to process a chain of blocks, writing them into
// the database if successful.
func testBlockChainImport(chain []*types.Block, processor *BlockProcessor) error {
for _, block := range chain {
// Try and process the block
if _, _, err := processor.Process(block); err != nil {
if IsKnownBlockErr(err) {
continue
}
return nil, err
return err
}
bman.bc.mu.Lock()
WriteTd(bman.bc.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), bman.bc.GetTd(block.ParentHash())))
WriteBlock(bman.bc.chainDb, block)
bman.bc.mu.Unlock()
// Manually insert the block into the database, but don't reorganize (allows subsequent testing)
processor.bc.mu.Lock()
WriteTd(processor.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), processor.bc.GetTd(block.ParentHash())))
WriteBlock(processor.chainDb, block)
processor.bc.mu.Unlock()
}
return bman.bc.GetTd(chainB[len(chainB)-1].Hash()), nil
return nil
}
// testHeaderChainImport tries to process a chain of header, writing them into
// the database if successful.
func testHeaderChainImport(chain []*types.Header, processor *BlockProcessor) error {
for _, header := range chain {
// Try and validate the header
if err := processor.ValidateHeader(header, false, false); err != nil {
return err
}
// Manually insert the header into the database, but don't reorganize (allows subsequent testing)
processor.bc.mu.Lock()
WriteTd(processor.chainDb, header.Hash(), new(big.Int).Add(header.Difficulty, processor.bc.GetTd(header.ParentHash)))
WriteHeader(processor.chainDb, header)
processor.bc.mu.Unlock()
}
return nil
}
func loadChain(fn string, t *testing.T) (types.Blocks, error) {
@ -153,127 +186,137 @@ func insertChain(done chan bool, chainMan *ChainManager, chain types.Blocks, t *
done <- true
}
func TestExtendCanonical(t *testing.T) {
CanonicalLength := 5
db, err := ethdb.NewMemDatabase()
// Tests that given a starting canonical chain of a given size, it can be extended
// with various length chains.
func TestExtendCanonicalHeaders(t *testing.T) { testExtendCanonical(t, false) }
func TestExtendCanonicalBlocks(t *testing.T) { testExtendCanonical(t, true) }
func testExtendCanonical(t *testing.T, full bool) {
length := 5
// Make first chain starting from genesis
_, processor, err := newCanonical(length, full)
if err != nil {
t.Fatal("Failed to create db:", err)
t.Fatalf("failed to make new canonical chain: %v", err)
}
// make first chain starting from genesis
bman, err := newCanonical(CanonicalLength, db)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
f := func(td1, td2 *big.Int) {
// Define the difficulty comparator
better := func(td1, td2 *big.Int) {
if td2.Cmp(td1) <= 0 {
t.Error("expected chainB to have higher difficulty. Got", td2, "expected more than", td1)
t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1)
}
}
// Start fork from current height (CanonicalLength)
testFork(t, bman, CanonicalLength, 1, f)
testFork(t, bman, CanonicalLength, 2, f)
testFork(t, bman, CanonicalLength, 5, f)
testFork(t, bman, CanonicalLength, 10, f)
// Start fork from current height
testFork(t, processor, length, 1, full, better)
testFork(t, processor, length, 2, full, better)
testFork(t, processor, length, 5, full, better)
testFork(t, processor, length, 10, full, better)
}
func TestShorterFork(t *testing.T) {
db, err := ethdb.NewMemDatabase()
// Tests that given a starting canonical chain of a given size, creating shorter
// forks do not take canonical ownership.
func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false) }
func TestShorterForkBlocks(t *testing.T) { testShorterFork(t, true) }
func testShorterFork(t *testing.T, full bool) {
length := 10
// Make first chain starting from genesis
_, processor, err := newCanonical(length, full)
if err != nil {
t.Fatal("Failed to create db:", err)
t.Fatalf("failed to make new canonical chain: %v", err)
}
// make first chain starting from genesis
bman, err := newCanonical(10, db)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
f := func(td1, td2 *big.Int) {
// Define the difficulty comparator
worse := func(td1, td2 *big.Int) {
if td2.Cmp(td1) >= 0 {
t.Error("expected chainB to have lower difficulty. Got", td2, "expected less than", td1)
t.Errorf("total difficulty mismatch: have %v, expected less than %v", td2, td1)
}
}
// Sum of numbers must be less than 10
// for this to be a shorter fork
testFork(t, bman, 0, 3, f)
testFork(t, bman, 0, 7, f)
testFork(t, bman, 1, 1, f)
testFork(t, bman, 1, 7, f)
testFork(t, bman, 5, 3, f)
testFork(t, bman, 5, 4, f)
// Sum of numbers must be less than `length` for this to be a shorter fork
testFork(t, processor, 0, 3, full, worse)
testFork(t, processor, 0, 7, full, worse)
testFork(t, processor, 1, 1, full, worse)
testFork(t, processor, 1, 7, full, worse)
testFork(t, processor, 5, 3, full, worse)
testFork(t, processor, 5, 4, full, worse)
}
func TestLongerFork(t *testing.T) {
db, err := ethdb.NewMemDatabase()
// Tests that given a starting canonical chain of a given size, creating longer
// forks do take canonical ownership.
func TestLongerForkHeaders(t *testing.T) { testLongerFork(t, false) }
func TestLongerForkBlocks(t *testing.T) { testLongerFork(t, true) }
func testLongerFork(t *testing.T, full bool) {
length := 10
// Make first chain starting from genesis
_, processor, err := newCanonical(length, full)
if err != nil {
t.Fatal("Failed to create db:", err)
t.Fatalf("failed to make new canonical chain: %v", err)
}
// make first chain starting from genesis
bman, err := newCanonical(10, db)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
f := func(td1, td2 *big.Int) {
// Define the difficulty comparator
better := func(td1, td2 *big.Int) {
if td2.Cmp(td1) <= 0 {
t.Error("expected chainB to have higher difficulty. Got", td2, "expected more than", td1)
t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1)
}
}
// Sum of numbers must be greater than 10
// for this to be a longer fork
testFork(t, bman, 0, 11, f)
testFork(t, bman, 0, 15, f)
testFork(t, bman, 1, 10, f)
testFork(t, bman, 1, 12, f)
testFork(t, bman, 5, 6, f)
testFork(t, bman, 5, 8, f)
// Sum of numbers must be greater than `length` for this to be a longer fork
testFork(t, processor, 0, 11, full, better)
testFork(t, processor, 0, 15, full, better)
testFork(t, processor, 1, 10, full, better)
testFork(t, processor, 1, 12, full, better)
testFork(t, processor, 5, 6, full, better)
testFork(t, processor, 5, 8, full, better)
}
func TestEqualFork(t *testing.T) {
db, err := ethdb.NewMemDatabase()
// Tests that given a starting canonical chain of a given size, creating equal
// forks do take canonical ownership.
func TestEqualForkHeaders(t *testing.T) { testEqualFork(t, false) }
func TestEqualForkBlocks(t *testing.T) { testEqualFork(t, true) }
func testEqualFork(t *testing.T, full bool) {
length := 10
// Make first chain starting from genesis
_, processor, err := newCanonical(length, full)
if err != nil {
t.Fatal("Failed to create db:", err)
t.Fatalf("failed to make new canonical chain: %v", err)
}
bman, err := newCanonical(10, db)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
f := func(td1, td2 *big.Int) {
// Define the difficulty comparator
equal := func(td1, td2 *big.Int) {
if td2.Cmp(td1) != 0 {
t.Error("expected chainB to have equal difficulty. Got", td2, "expected ", td1)
t.Errorf("total difficulty mismatch: have %v, want %v", td2, td1)
}
}
// Sum of numbers must be equal to 10
// for this to be an equal fork
testFork(t, bman, 0, 10, f)
testFork(t, bman, 1, 9, f)
testFork(t, bman, 2, 8, f)
testFork(t, bman, 5, 5, f)
testFork(t, bman, 6, 4, f)
testFork(t, bman, 9, 1, f)
// Sum of numbers must be equal to `length` for this to be an equal fork
testFork(t, processor, 0, 10, full, equal)
testFork(t, processor, 1, 9, full, equal)
testFork(t, processor, 2, 8, full, equal)
testFork(t, processor, 5, 5, full, equal)
testFork(t, processor, 6, 4, full, equal)
testFork(t, processor, 9, 1, full, equal)
}
func TestBrokenChain(t *testing.T) {
db, err := ethdb.NewMemDatabase()
// Tests that chains missing links do not get accepted by the processor.
func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false) }
func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true) }
func testBrokenChain(t *testing.T, full bool) {
// Make chain starting from genesis
db, processor, err := newCanonical(10, full)
if err != nil {
t.Fatal("Failed to create db:", err)
t.Fatalf("failed to make new canonical chain: %v", err)
}
bman, err := newCanonical(10, db)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
db2, err := ethdb.NewMemDatabase()
if err != nil {
t.Fatal("Failed to create db:", err)
}
bman2, err := newCanonical(10, db2)
if err != nil {
t.Fatal("Could not make new canonical chain:", err)
}
bman2.bc.SetProcessor(bman2)
parent := bman2.bc.CurrentBlock()
chainB := makeChain(parent, 5, db2, forkSeed)
chainB = chainB[1:]
_, err = testChain(chainB, bman)
if err == nil {
t.Error("expected broken chain to return error")
// Create a forked chain, and try to insert with a missing link
if full {
chain := makeBlockChain(processor.bc.CurrentBlock(), 5, db, forkSeed)[1:]
if err := testBlockChainImport(chain, processor); err == nil {
t.Errorf("broken block chain not reported")
}
} else {
chain := makeHeaderChain(processor.bc.CurrentHeader(), 5, db, forkSeed)[1:]
if err := testHeaderChainImport(chain, processor); err == nil {
t.Errorf("broken header chain not reported")
}
}
}
@ -363,13 +406,25 @@ type bproc struct{}
func (bproc) Process(*types.Block) (state.Logs, types.Receipts, error) { return nil, nil, nil }
func makeChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header {
blocks := makeBlockChainWithDiff(genesis, d, seed)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
return headers
}
func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
var chain []*types.Block
for i, difficulty := range d {
header := &types.Header{
Coinbase: common.Address{seed},
Number: big.NewInt(int64(i + 1)),
Difficulty: big.NewInt(int64(difficulty)),
Coinbase: common.Address{seed},
Number: big.NewInt(int64(i + 1)),
Difficulty: big.NewInt(int64(difficulty)),
UncleHash: types.EmptyUncleHash,
TxHash: types.EmptyRootHash,
ReceiptHash: types.EmptyRootHash,
}
if i == 0 {
header.ParentHash = genesis.Hash()
@ -397,147 +452,363 @@ func chm(genesis *types.Block, db ethdb.Database) *ChainManager {
return bc
}
func TestReorgLongest(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
// Tests that reorganizing a long difficult chain after a short easy one
// overwrites the canonical numbers and links in the database.
func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) }
func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) }
genesis, err := WriteTestNetGenesisBlock(db, 0)
if err != nil {
t.Error(err)
t.FailNow()
}
func testReorgLong(t *testing.T, full bool) {
testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full)
}
// Tests that reorganizing a short difficult chain after a long easy one
// overwrites the canonical numbers and links in the database.
func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) }
func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) }
func testReorgShort(t *testing.T, full bool) {
testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full)
}
func testReorg(t *testing.T, first, second []int, td int64, full bool) {
// Create a pristine block chain
db, _ := ethdb.NewMemDatabase()
genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
chain1 := makeChainWithDiff(genesis, []int{1, 2, 4}, 10)
chain2 := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 11)
bc.InsertChain(chain1)
bc.InsertChain(chain2)
prev := bc.CurrentBlock()
for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
if prev.ParentHash() != block.Hash() {
t.Errorf("parent hash mismatch %x - %x", prev.ParentHash(), block.Hash())
// Insert an easy and a difficult chain afterwards
if full {
bc.InsertChain(makeBlockChainWithDiff(genesis, first, 11))
bc.InsertChain(makeBlockChainWithDiff(genesis, second, 22))
} else {
bc.InsertHeaderChain(makeHeaderChainWithDiff(genesis, first, 11), false)
bc.InsertHeaderChain(makeHeaderChainWithDiff(genesis, second, 22), false)
}
// Check that the chain is valid number and link wise
if full {
prev := bc.CurrentBlock()
for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
if prev.ParentHash() != block.Hash() {
t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash())
}
}
} else {
prev := bc.CurrentHeader()
for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) {
if prev.ParentHash != header.Hash() {
t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash())
}
}
}
// Make sure the chain total difficulty is the correct one
want := new(big.Int).Add(genesis.Difficulty(), big.NewInt(td))
if full {
if have := bc.GetTd(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 {
t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
}
} else {
if have := bc.GetTd(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 {
t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
}
}
}
func TestBadHashes(t *testing.T) {
// Tests that the insertion functions detect banned hashes.
func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) }
func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) }
func testBadHashes(t *testing.T, full bool) {
// Create a pristine block chain
db, _ := ethdb.NewMemDatabase()
genesis, err := WriteTestNetGenesisBlock(db, 0)
if err != nil {
t.Error(err)
t.FailNow()
}
genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
chain := makeChainWithDiff(genesis, []int{1, 2, 4}, 10)
BadHashes[chain[2].Header().Hash()] = true
_, err = bc.InsertChain(chain)
// Create a chain, ban a hash and try to import
var err error
if full {
blocks := makeBlockChainWithDiff(genesis, []int{1, 2, 4}, 10)
BadHashes[blocks[2].Header().Hash()] = true
_, err = bc.InsertChain(blocks)
} else {
headers := makeHeaderChainWithDiff(genesis, []int{1, 2, 4}, 10)
BadHashes[headers[2].Hash()] = true
_, err = bc.InsertHeaderChain(headers, true)
}
if !IsBadHashError(err) {
t.Errorf("error mismatch: want: BadHashError, have: %v", err)
}
}
func TestReorgBadHashes(t *testing.T) {
// Tests that bad hashes are detected on boot, and the chan rolled back to a
// good state prior to the bad hash.
func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) }
func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) }
func testReorgBadHashes(t *testing.T, full bool) {
// Create a pristine block chain
db, _ := ethdb.NewMemDatabase()
genesis, err := WriteTestNetGenesisBlock(db, 0)
if err != nil {
t.Error(err)
t.FailNow()
}
genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
chain := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 11)
bc.InsertChain(chain)
// Create a chain, import and ban aferwards
headers := makeHeaderChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
blocks := makeBlockChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
if chain[3].Header().Hash() != bc.LastBlockHash() {
t.Errorf("last block hash mismatch: want: %x, have: %x", chain[3].Header().Hash(), bc.LastBlockHash())
if full {
if _, err := bc.InsertChain(blocks); err != nil {
t.Fatalf("failed to import blocks: %v", err)
}
if bc.CurrentBlock().Hash() != blocks[3].Hash() {
t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash())
}
BadHashes[blocks[3].Header().Hash()] = true
defer func() { delete(BadHashes, blocks[3].Header().Hash()) }()
} else {
if _, err := bc.InsertHeaderChain(headers, true); err != nil {
t.Fatalf("failed to import headers: %v", err)
}
if bc.CurrentHeader().Hash() != headers[3].Hash() {
t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash())
}
BadHashes[headers[3].Hash()] = true
defer func() { delete(BadHashes, headers[3].Hash()) }()
}
// NewChainManager should check BadHashes when loading it db
BadHashes[chain[3].Header().Hash()] = true
var eventMux event.TypeMux
ncm, err := NewChainManager(db, FakePow{}, &eventMux)
// Create a new chain manager and check it rolled back the state
ncm, err := NewChainManager(db, FakePow{}, new(event.TypeMux))
if err != nil {
t.Errorf("NewChainManager err: %s", err)
t.Fatalf("failed to create new chain manager: %v", err)
}
// check it set head to (valid) parent of bad hash block
if chain[2].Header().Hash() != ncm.LastBlockHash() {
t.Errorf("last block hash mismatch: want: %x, have: %x", chain[2].Header().Hash(), ncm.LastBlockHash())
}
if chain[2].Header().GasLimit.Cmp(ncm.GasLimit()) != 0 {
t.Errorf("current block gasLimit mismatch: want: %x, have: %x", chain[2].Header().GasLimit, ncm.GasLimit())
}
}
func TestReorgShortest(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
genesis, err := WriteTestNetGenesisBlock(db, 0)
if err != nil {
t.Error(err)
t.FailNow()
}
bc := chm(genesis, db)
chain1 := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
chain2 := makeChainWithDiff(genesis, []int{1, 10}, 11)
bc.InsertChain(chain1)
bc.InsertChain(chain2)
prev := bc.CurrentBlock()
for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
if prev.ParentHash() != block.Hash() {
t.Errorf("parent hash mismatch %x - %x", prev.ParentHash(), block.Hash())
if full {
if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() {
t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash())
}
if blocks[2].Header().GasLimit.Cmp(ncm.GasLimit()) != 0 {
t.Errorf("last block gasLimit mismatch: have: %x, want %x", ncm.GasLimit(), blocks[2].Header().GasLimit)
}
} else {
if ncm.CurrentHeader().Hash() != genesis.Hash() {
t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), genesis.Hash())
}
}
}
func TestInsertNonceError(t *testing.T) {
// Tests chain insertions in the face of one entity containing an invalid nonce.
func TestHeadersInsertNonceError(t *testing.T) { testInsertNonceError(t, false) }
func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true) }
func testInsertNonceError(t *testing.T, full bool) {
for i := 1; i < 25 && !t.Failed(); i++ {
db, _ := ethdb.NewMemDatabase()
genesis, err := WriteTestNetGenesisBlock(db, 0)
// Create a pristine chain and database
db, processor, err := newCanonical(0, full)
if err != nil {
t.Error(err)
t.FailNow()
t.Fatalf("failed to create pristine chain: %v", err)
}
bc := chm(genesis, db)
bc.processor = NewBlockProcessor(db, bc.pow, bc, bc.eventMux)
blocks := makeChain(bc.currentBlock, i, db, 0)
bc := processor.bc
fail := rand.Int() % len(blocks)
failblock := blocks[fail]
bc.pow = failPow{failblock.NumberU64()}
n, err := bc.InsertChain(blocks)
// Create and insert a chain with a failing nonce
var (
failAt int
failRes int
failNum uint64
failHash common.Hash
)
if full {
blocks := makeBlockChain(processor.bc.CurrentBlock(), i, db, 0)
failAt = rand.Int() % len(blocks)
failNum = blocks[failAt].NumberU64()
failHash = blocks[failAt].Hash()
processor.bc.pow = failPow{failNum}
failRes, err = processor.bc.InsertChain(blocks)
} else {
headers := makeHeaderChain(processor.bc.CurrentHeader(), i, db, 0)
failAt = rand.Int() % len(headers)
failNum = headers[failAt].Number.Uint64()
failHash = headers[failAt].Hash()
processor.bc.pow = failPow{failNum}
failRes, err = processor.bc.InsertHeaderChain(headers, true)
}
// Check that the returned error indicates the nonce failure.
if n != fail {
t.Errorf("(i=%d) wrong failed block index: got %d, want %d", i, n, fail)
if failRes != failAt {
t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
}
if !IsBlockNonceErr(err) {
t.Fatalf("(i=%d) got %q, want a nonce error", i, err)
t.Fatalf("test %d: error mismatch: have %v, want nonce error", i, err)
}
nerr := err.(*BlockNonceErr)
if nerr.Number.Cmp(failblock.Number()) != 0 {
t.Errorf("(i=%d) wrong block number in error, got %v, want %v", i, nerr.Number, failblock.Number())
if nerr.Number.Uint64() != failNum {
t.Errorf("test %d: number mismatch: have %v, want %v", i, nerr.Number, failNum)
}
if nerr.Hash != failblock.Hash() {
t.Errorf("(i=%d) wrong block hash in error, got %v, want %v", i, nerr.Hash, failblock.Hash())
if nerr.Hash != failHash {
t.Errorf("test %d: hash mismatch: have %x, want %x", i, nerr.Hash[:4], failHash[:4])
}
// Check that all no blocks after the failing block have been inserted.
for _, block := range blocks[fail:] {
if bc.HasBlock(block.Hash()) {
t.Errorf("(i=%d) invalid block %d present in chain", i, block.NumberU64())
for j := 0; j < i-failAt; j++ {
if full {
if block := bc.GetBlockByNumber(failNum + uint64(j)); block != nil {
t.Errorf("test %d: invalid block in chain: %v", i, block)
}
} else {
if header := bc.GetHeaderByNumber(failNum + uint64(j)); header != nil {
t.Errorf("test %d: invalid header in chain: %v", i, header)
}
}
}
}
}
// Tests that fast importing a block chain produces the same chain data as the
// classical full block processing.
func TestFastVsFullChains(t *testing.T) {
// Configure and generate a sample block chain
var (
gendb, _ = ethdb.NewMemDatabase()
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
genesis = GenesisBlockForTesting(gendb, address, funds)
)
blocks, receipts := GenerateChain(genesis, gendb, 1024, func(i int, block *BlockGen) {
block.SetCoinbase(common.Address{0x00})
// If the block number is multiple of 3, send a few bonus transactions to the miner
if i%3 == 2 {
for j := 0; j < i%4+1; j++ {
tx, err := types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key)
if err != nil {
panic(err)
}
block.AddTx(tx)
}
}
// If the block number is a multiple of 5, add a few bonus uncles to the block
if i%5 == 5 {
block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))})
}
})
// Import the chain as an archive node for the comparison baseline
archiveDb, _ := ethdb.NewMemDatabase()
WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
archive, _ := NewChainManager(archiveDb, FakePow{}, new(event.TypeMux))
archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux)))
if n, err := archive.InsertChain(blocks); err != nil {
t.Fatalf("failed to process block %d: %v", n, err)
}
// Fast import the chain as a non-archive node to test
fastDb, _ := ethdb.NewMemDatabase()
WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
fast, _ := NewChainManager(fastDb, FakePow{}, new(event.TypeMux))
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers, true); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// Iterate over all chain data components, and cross reference
for i := 0; i < len(blocks); i++ {
num, hash := blocks[i].NumberU64(), blocks[i].Hash()
if ftd, atd := fast.GetTd(hash), archive.GetTd(hash); ftd.Cmp(atd) != 0 {
t.Errorf("block #%d [%x]: td mismatch: have %v, want %v", num, hash, ftd, atd)
}
if fheader, aheader := fast.GetHeader(hash), archive.GetHeader(hash); fheader.Hash() != aheader.Hash() {
t.Errorf("block #%d [%x]: header mismatch: have %v, want %v", num, hash, fheader, aheader)
}
if fblock, ablock := fast.GetBlock(hash), archive.GetBlock(hash); fblock.Hash() != ablock.Hash() {
t.Errorf("block #%d [%x]: block mismatch: have %v, want %v", num, hash, fblock, ablock)
} else if types.DeriveSha(fblock.Transactions()) != types.DeriveSha(ablock.Transactions()) {
t.Errorf("block #%d [%x]: transactions mismatch: have %v, want %v", num, hash, fblock.Transactions(), ablock.Transactions())
} else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) {
t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles())
}
if freceipts, areceipts := GetBlockReceipts(fastDb, hash), GetBlockReceipts(archiveDb, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) {
t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts)
}
}
// Check that the canonical chains are the same between the databases
for i := 0; i < len(blocks)+1; i++ {
if fhash, ahash := GetCanonicalHash(fastDb, uint64(i)), GetCanonicalHash(archiveDb, uint64(i)); fhash != ahash {
t.Errorf("block #%d: canonical hash mismatch: have %v, want %v", i, fhash, ahash)
}
}
}
// Tests that various import methods move the chain head pointers to the correct
// positions.
func TestLightVsFastVsFullChainHeads(t *testing.T) {
// Configure and generate a sample block chain
var (
gendb, _ = ethdb.NewMemDatabase()
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000)
genesis = GenesisBlockForTesting(gendb, address, funds)
)
height := uint64(1024)
blocks, receipts := GenerateChain(genesis, gendb, int(height), nil)
// Create a small assertion method to check the three heads
assert := func(t *testing.T, kind string, chain *ChainManager, header uint64, fast uint64, block uint64) {
if num := chain.CurrentBlock().NumberU64(); num != block {
t.Errorf("%s head block mismatch: have #%v, want #%v", kind, num, block)
}
if num := chain.CurrentFastBlock().NumberU64(); num != fast {
t.Errorf("%s head fast-block mismatch: have #%v, want #%v", kind, num, fast)
}
if num := chain.CurrentHeader().Number.Uint64(); num != header {
t.Errorf("%s head header mismatch: have #%v, want #%v", kind, num, header)
}
}
// Import the chain as an archive node and ensure all pointers are updated
archiveDb, _ := ethdb.NewMemDatabase()
WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
archive, _ := NewChainManager(archiveDb, FakePow{}, new(event.TypeMux))
archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux)))
if n, err := archive.InsertChain(blocks); err != nil {
t.Fatalf("failed to process block %d: %v", n, err)
}
assert(t, "archive", archive, height, height, height)
// Import the chain as a non-archive node and ensure all pointers are updated
fastDb, _ := ethdb.NewMemDatabase()
WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
fast, _ := NewChainManager(fastDb, FakePow{}, new(event.TypeMux))
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers, true); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
assert(t, "fast", fast, height, height, 0)
// Import the chain as a light node and ensure all pointers are updated
lightDb, _ := ethdb.NewMemDatabase()
WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds})
light, _ := NewChainManager(lightDb, FakePow{}, new(event.TypeMux))
if n, err := light.InsertHeaderChain(headers, true); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
assert(t, "light", light, height, 0, 0)
}
// Tests that chain reorganizations handle transaction removals and reinsertions.
func TestChainTxReorgs(t *testing.T) {
params.MinGasLimit = big.NewInt(125000) // Minimum the gas limit may ever be.
@ -574,7 +845,7 @@ func TestChainTxReorgs(t *testing.T) {
// - futureAdd: transaction added after the reorg has already finished
var pastAdd, freshAdd, futureAdd *types.Transaction
chain := GenerateChain(genesis, db, 3, func(i int, gen *BlockGen) {
chain, _ := GenerateChain(genesis, db, 3, func(i int, gen *BlockGen) {
switch i {
case 0:
pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key2)
@ -600,7 +871,7 @@ func TestChainTxReorgs(t *testing.T) {
}
// overwrite the old chain
chain = GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
chain, _ = GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key3)

6
core/chain_pow_test.go
View file

@ -60,7 +60,7 @@ func TestPowVerification(t *testing.T) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
blocks = GenerateChain(genesis, testdb, 8, nil)
blocks, _ = GenerateChain(genesis, testdb, 8, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
@ -115,7 +115,7 @@ func testPowConcurrentVerification(t *testing.T, threads int) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
blocks = GenerateChain(genesis, testdb, 8, nil)
blocks, _ = GenerateChain(genesis, testdb, 8, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
@ -186,7 +186,7 @@ func testPowConcurrentAbortion(t *testing.T, threads int) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
blocks = GenerateChain(genesis, testdb, 1024, nil)
blocks, _ = GenerateChain(genesis, testdb, 1024, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {

24
core/chain_util.go
View file

@ -32,6 +32,7 @@ import (
var (
headHeaderKey = []byte("LastHeader")
headBlockKey = []byte("LastBlock")
headFastKey = []byte("LastFast")
blockPrefix = []byte("block-")
blockNumPrefix = []byte("block-num-")
@ -124,7 +125,7 @@ func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash {
// header. The difference between this and GetHeadBlockHash is that whereas the
// last block hash is only updated upon a full block import, the last header
// hash is updated already at header import, allowing head tracking for the
// fast synchronization mechanism.
// light synchronization mechanism.
func GetHeadHeaderHash(db ethdb.Database) common.Hash {
data, _ := db.Get(headHeaderKey)
if len(data) == 0 {
@ -142,6 +143,18 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash {
return common.BytesToHash(data)
}
// GetHeadFastBlockHash retrieves the hash of the current canonical head block during
// fast synchronization. The difference between this and GetHeadBlockHash is that
// whereas the last block hash is only updated upon a full block import, the last
// fast hash is updated when importing pre-processed blocks.
func GetHeadFastBlockHash(db ethdb.Database) common.Hash {
data, _ := db.Get(headFastKey)
if len(data) == 0 {
return common.Hash{}
}
return common.BytesToHash(data)
}
// GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil
// if the header's not found.
func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue {
@ -244,6 +257,15 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error {
return nil
}
// WriteHeadFastBlockHash stores the fast head block's hash.
func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error {
if err := db.Put(headFastKey, hash.Bytes()); err != nil {
glog.Fatalf("failed to store last fast block's hash into database: %v", err)
return err
}
return nil
}
// WriteHeader serializes a block header into the database.
func WriteHeader(db ethdb.Database, header *types.Header) error {
data, err := rlp.EncodeToBytes(header)

25
core/chain_util_test.go
View file

@ -160,7 +160,12 @@ func TestBlockStorage(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
// Create a test block to move around the database and make sure it's really new
block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")})
block := types.NewBlockWithHeader(&types.Header{
Extra: []byte("test block"),
UncleHash: types.EmptyUncleHash,
TxHash: types.EmptyRootHash,
ReceiptHash: types.EmptyRootHash,
})
if entry := GetBlock(db, block.Hash()); entry != nil {
t.Fatalf("Non existent block returned: %v", entry)
}
@ -205,8 +210,12 @@ func TestBlockStorage(t *testing.T) {
// Tests that partial block contents don't get reassembled into full blocks.
func TestPartialBlockStorage(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")})
block := types.NewBlockWithHeader(&types.Header{
Extra: []byte("test block"),
UncleHash: types.EmptyUncleHash,
TxHash: types.EmptyRootHash,
ReceiptHash: types.EmptyRootHash,
})
// Store a header and check that it's not recognized as a block
if err := WriteHeader(db, block.Header()); err != nil {
t.Fatalf("Failed to write header into database: %v", err)
@ -295,6 +304,7 @@ func TestHeadStorage(t *testing.T) {
blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")})
blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")})
blockFast := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block fast")})
// Check that no head entries are in a pristine database
if entry := GetHeadHeaderHash(db); entry != (common.Hash{}) {
@ -303,6 +313,9 @@ func TestHeadStorage(t *testing.T) {
if entry := GetHeadBlockHash(db); entry != (common.Hash{}) {
t.Fatalf("Non head block entry returned: %v", entry)
}
if entry := GetHeadFastBlockHash(db); entry != (common.Hash{}) {
t.Fatalf("Non fast head block entry returned: %v", entry)
}
// Assign separate entries for the head header and block
if err := WriteHeadHeaderHash(db, blockHead.Hash()); err != nil {
t.Fatalf("Failed to write head header hash: %v", err)
@ -310,6 +323,9 @@ func TestHeadStorage(t *testing.T) {
if err := WriteHeadBlockHash(db, blockFull.Hash()); err != nil {
t.Fatalf("Failed to write head block hash: %v", err)
}
if err := WriteHeadFastBlockHash(db, blockFast.Hash()); err != nil {
t.Fatalf("Failed to write fast head block hash: %v", err)
}
// Check that both heads are present, and different (i.e. two heads maintained)
if entry := GetHeadHeaderHash(db); entry != blockHead.Hash() {
t.Fatalf("Head header hash mismatch: have %v, want %v", entry, blockHead.Hash())
@ -317,4 +333,7 @@ func TestHeadStorage(t *testing.T) {
if entry := GetHeadBlockHash(db); entry != blockFull.Hash() {
t.Fatalf("Head block hash mismatch: have %v, want %v", entry, blockFull.Hash())
}
if entry := GetHeadFastBlockHash(db); entry != blockFast.Hash() {
t.Fatalf("Fast head block hash mismatch: have %v, want %v", entry, blockFast.Hash())
}
}

3
core/genesis.go
View file

@ -102,6 +102,9 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block,
if err := WriteBlock(chainDb, block); err != nil {
return nil, err
}
if err := PutBlockReceipts(chainDb, block.Hash(), nil); err != nil {
return nil, err
}
if err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()); err != nil {
return nil, err
}

51
core/state/log.go
View file

@ -25,42 +25,47 @@ import (
)
type Log struct {
// Consensus fields
Address common.Address
Topics []common.Hash
Data []byte
Number uint64
TxHash common.Hash
TxIndex uint
BlockHash common.Hash
Index uint
// Derived fields (don't reorder!)
BlockNumber uint64
TxHash common.Hash
TxIndex uint
BlockHash common.Hash
Index uint
}
func NewLog(address common.Address, topics []common.Hash, data []byte, number uint64) *Log {
return &Log{Address: address, Topics: topics, Data: data, Number: number}
return &Log{Address: address, Topics: topics, Data: data, BlockNumber: number}
}
func (self *Log) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{self.Address, self.Topics, self.Data})
func (l *Log) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{l.Address, l.Topics, l.Data})
}
func (self *Log) String() string {
return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, self.Address, self.Topics, self.Data, self.TxHash, self.TxIndex, self.BlockHash, self.Index)
func (l *Log) DecodeRLP(s *rlp.Stream) error {
var log struct {
Address common.Address
Topics []common.Hash
Data []byte
}
if err := s.Decode(&log); err != nil {
return err
}
l.Address, l.Topics, l.Data = log.Address, log.Topics, log.Data
return nil
}
func (l *Log) String() string {
return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
}
type Logs []*Log
// LogForStorage is a wrapper around a Log that flattens and parses the entire
// content of a log, opposed to only the consensus fields originally (by hiding
// the rlp interface methods).
type LogForStorage Log
func (self *LogForStorage) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{
self.Address,
self.Topics,
self.Data,
self.Number,
self.TxHash,
self.TxIndex,
self.BlockHash,
self.Index,
})
}

18
core/transaction_util.go
View file

@ -150,11 +150,14 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
if len(data) == 0 {
return nil
}
var receipts types.Receipts
err := rlp.DecodeBytes(data, &receipts)
if err != nil {
glog.V(logger.Core).Infoln("GetReceiptse err", err)
rs := []*types.ReceiptForStorage{}
if err := rlp.DecodeBytes(data, &rs); err != nil {
glog.V(logger.Error).Infof("invalid receipt array RLP for hash %x: %v", hash, err)
return nil
}
receipts := make(types.Receipts, len(rs))
for i, receipt := range rs {
receipts[i] = (*types.Receipt)(receipt)
}
return receipts
}
@ -162,7 +165,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
// PutBlockReceipts stores the block's transactions associated receipts
// and stores them by block hash in a single slice. This is required for
// forks and chain reorgs
func PutBlockReceipts(db ethdb.Database, block *types.Block, receipts types.Receipts) error {
func PutBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error {
rs := make([]*types.ReceiptForStorage, len(receipts))
for i, receipt := range receipts {
rs[i] = (*types.ReceiptForStorage)(receipt)
@ -171,12 +174,9 @@ func PutBlockReceipts(db ethdb.Database, block *types.Block, receipts types.Rece
if err != nil {
return err
}
hash := block.Hash()
err = db.Put(append(blockReceiptsPre, hash[:]...), bytes)
if err != nil {
return err
}
return nil
}

31
core/types/block.go
View file

@ -128,7 +128,6 @@ type Block struct {
header *Header
uncles []*Header
transactions Transactions
receipts Receipts
// caches
hash atomic.Value
@ -172,8 +171,8 @@ type storageblock struct {
}
var (
emptyRootHash = DeriveSha(Transactions{})
emptyUncleHash = CalcUncleHash(nil)
EmptyRootHash = DeriveSha(Transactions{})
EmptyUncleHash = CalcUncleHash(nil)
)
// NewBlock creates a new block. The input data is copied,
@ -184,11 +183,11 @@ var (
// are ignored and set to values derived from the given txs, uncles
// and receipts.
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt) *Block {
b := &Block{header: copyHeader(header), td: new(big.Int)}
b := &Block{header: CopyHeader(header), td: new(big.Int)}
// TODO: panic if len(txs) != len(receipts)
if len(txs) == 0 {
b.header.TxHash = emptyRootHash
b.header.TxHash = EmptyRootHash
} else {
b.header.TxHash = DeriveSha(Transactions(txs))
b.transactions = make(Transactions, len(txs))
@ -196,21 +195,19 @@ func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*
}
if len(receipts) == 0 {
b.header.ReceiptHash = emptyRootHash
b.header.ReceiptHash = EmptyRootHash
} else {
b.header.ReceiptHash = DeriveSha(Receipts(receipts))
b.header.Bloom = CreateBloom(receipts)
b.receipts = make([]*Receipt, len(receipts))
copy(b.receipts, receipts)
}
if len(uncles) == 0 {
b.header.UncleHash = emptyUncleHash
b.header.UncleHash = EmptyUncleHash
} else {
b.header.UncleHash = CalcUncleHash(uncles)
b.uncles = make([]*Header, len(uncles))
for i := range uncles {
b.uncles[i] = copyHeader(uncles[i])
b.uncles[i] = CopyHeader(uncles[i])
}
}
@ -221,10 +218,12 @@ func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*
// header data is copied, changes to header and to the field values
// will not affect the block.
func NewBlockWithHeader(header *Header) *Block {
return &Block{header: copyHeader(header)}
return &Block{header: CopyHeader(header)}
}
func copyHeader(h *Header) *Header {
// CopyHeader creates a deep copy of a block header to prevent side effects from
// modifying a header variable.
func CopyHeader(h *Header) *Header {
cpy := *h
if cpy.Time = new(big.Int); h.Time != nil {
cpy.Time.Set(h.Time)
@ -297,7 +296,6 @@ func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error {
// TODO: copies
func (b *Block) Uncles() []*Header { return b.uncles }
func (b *Block) Transactions() Transactions { return b.transactions }
func (b *Block) Receipts() Receipts { return b.receipts }
func (b *Block) Transaction(hash common.Hash) *Transaction {
for _, transaction := range b.transactions {
@ -326,7 +324,7 @@ func (b *Block) ReceiptHash() common.Hash { return b.header.ReceiptHash }
func (b *Block) UncleHash() common.Hash { return b.header.UncleHash }
func (b *Block) Extra() []byte { return common.CopyBytes(b.header.Extra) }
func (b *Block) Header() *Header { return copyHeader(b.header) }
func (b *Block) Header() *Header { return CopyHeader(b.header) }
func (b *Block) HashNoNonce() common.Hash {
return b.header.HashNoNonce()
@ -362,7 +360,6 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block {
return &Block{
header: &cpy,
transactions: b.transactions,
receipts: b.receipts,
uncles: b.uncles,
}
}
@ -370,13 +367,13 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block {
// WithBody returns a new block with the given transaction and uncle contents.
func (b *Block) WithBody(transactions []*Transaction, uncles []*Header) *Block {
block := &Block{
header: copyHeader(b.header),
header: CopyHeader(b.header),
transactions: make([]*Transaction, len(transactions)),
uncles: make([]*Header, len(uncles)),
}
copy(block.transactions, transactions)
for i := range uncles {
block.uncles[i] = copyHeader(uncles[i])
block.uncles[i] = CopyHeader(uncles[i])
}
return block
}

2
core/types/bloom9.go
View file

@ -31,7 +31,7 @@ type bytesBacked interface {
func CreateBloom(receipts Receipts) Bloom {
bin := new(big.Int)
for _, receipt := range receipts {
bin.Or(bin, LogsBloom(receipt.logs))
bin.Or(bin, LogsBloom(receipt.Logs))
}
return BytesToBloom(bin.Bytes())

132
core/types/receipt.go
View file

@ -17,7 +17,6 @@
package types
import (
"bytes"
"fmt"
"io"
"math/big"
@ -27,89 +26,116 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
// Receipt represents the results of a transaction.
type Receipt struct {
// Consensus fields
PostState []byte
CumulativeGasUsed *big.Int
Bloom Bloom
TxHash common.Hash
ContractAddress common.Address
logs state.Logs
GasUsed *big.Int
Logs state.Logs
// Implementation fields
TxHash common.Hash
ContractAddress common.Address
GasUsed *big.Int
}
func NewReceipt(root []byte, cumalativeGasUsed *big.Int) *Receipt {
return &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: new(big.Int).Set(cumalativeGasUsed)}
// NewReceipt creates a barebone transaction receipt, copying the init fields.
func NewReceipt(root []byte, cumulativeGasUsed *big.Int) *Receipt {
return &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: new(big.Int).Set(cumulativeGasUsed)}
}
func (self *Receipt) SetLogs(logs state.Logs) {
self.logs = logs
// EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt
// into an RLP stream.
func (r *Receipt) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs})
}
func (self *Receipt) Logs() state.Logs {
return self.logs
// DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt
// from an RLP stream.
func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
var receipt struct {
PostState []byte
CumulativeGasUsed *big.Int
Bloom Bloom
Logs state.Logs
}
if err := s.Decode(&receipt); err != nil {
return err
}
r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs = receipt.PostState, receipt.CumulativeGasUsed, receipt.Bloom, receipt.Logs
return nil
}
func (self *Receipt) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, []interface{}{self.PostState, self.CumulativeGasUsed, self.Bloom, self.logs})
// RlpEncode implements common.RlpEncode required for SHA derivation.
func (r *Receipt) RlpEncode() []byte {
bytes, err := rlp.EncodeToBytes(r)
if err != nil {
panic(err)
}
return bytes
}
func (self *Receipt) DecodeRLP(s *rlp.Stream) error {
var r struct {
// String implements the Stringer interface.
func (r *Receipt) String() string {
return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
}
// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
// entire content of a receipt, opposed to only the consensus fields originally.
type ReceiptForStorage Receipt
// EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt
// into an RLP stream.
func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
logs := make([]*state.LogForStorage, len(r.Logs))
for i, log := range r.Logs {
logs[i] = (*state.LogForStorage)(log)
}
return rlp.Encode(w, []interface{}{r.PostState, r.CumulativeGasUsed, r.Bloom, r.TxHash, r.ContractAddress, logs, r.GasUsed})
}
// DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt
// from an RLP stream.
func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
var receipt struct {
PostState []byte
CumulativeGasUsed *big.Int
Bloom Bloom
TxHash common.Hash
ContractAddress common.Address
Logs state.Logs
Logs []*state.LogForStorage
GasUsed *big.Int
}
if err := s.Decode(&r); err != nil {
if err := s.Decode(&receipt); err != nil {
return err
}
self.PostState, self.CumulativeGasUsed, self.Bloom, self.TxHash, self.ContractAddress, self.logs, self.GasUsed = r.PostState, r.CumulativeGasUsed, r.Bloom, r.TxHash, r.ContractAddress, r.Logs, r.GasUsed
// Assign the consensus fields
r.PostState, r.CumulativeGasUsed, r.Bloom = receipt.PostState, receipt.CumulativeGasUsed, receipt.Bloom
r.Logs = make(state.Logs, len(receipt.Logs))
for i, log := range receipt.Logs {
r.Logs[i] = (*state.Log)(log)
}
// Assign the implementation fields
r.TxHash, r.ContractAddress, r.GasUsed = receipt.TxHash, receipt.ContractAddress, receipt.GasUsed
return nil
}
type ReceiptForStorage Receipt
func (self *ReceiptForStorage) EncodeRLP(w io.Writer) error {
storageLogs := make([]*state.LogForStorage, len(self.logs))
for i, log := range self.logs {
storageLogs[i] = (*state.LogForStorage)(log)
}
return rlp.Encode(w, []interface{}{self.PostState, self.CumulativeGasUsed, self.Bloom, self.TxHash, self.ContractAddress, storageLogs, self.GasUsed})
}
func (self *Receipt) RlpEncode() []byte {
bytes, err := rlp.EncodeToBytes(self)
if err != nil {
fmt.Println("TMP -- RECEIPT ENCODE ERROR", err)
}
return bytes
}
func (self *Receipt) Cmp(other *Receipt) bool {
if bytes.Compare(self.PostState, other.PostState) != 0 {
return false
}
return true
}
func (self *Receipt) String() string {
return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", self.PostState, self.CumulativeGasUsed, self.Bloom, self.logs)
}
// Receipts is a wrapper around a Receipt array to implement types.DerivableList.
type Receipts []*Receipt
func (self Receipts) RlpEncode() []byte {
bytes, err := rlp.EncodeToBytes(self)
// RlpEncode implements common.RlpEncode required for SHA derivation.
func (r Receipts) RlpEncode() []byte {
bytes, err := rlp.EncodeToBytes(r)
if err != nil {
fmt.Println("TMP -- RECEIPTS ENCODE ERROR", err)
panic(err)
}
return bytes
}
func (self Receipts) Len() int { return len(self) }
func (self Receipts) GetRlp(i int) []byte { return common.Rlp(self[i]) }
// Len returns the number of receipts in this list.
func (r Receipts) Len() int { return len(r) }
// GetRlp returns the RLP encoding of one receipt from the list.
func (r Receipts) GetRlp(i int) []byte { return common.Rlp(r[i]) }

8
eth/backend.go
View file

@ -82,6 +82,7 @@ type Config struct {
GenesisFile string
GenesisBlock *types.Block // used by block tests
Olympic bool
Mode Mode
BlockChainVersion int
SkipBcVersionCheck bool // e.g. blockchain export
@ -377,8 +378,9 @@ func New(config *Config) (*Ethereum, error) {
eth.blockProcessor = core.NewBlockProcessor(chainDb, eth.pow, eth.chainManager, eth.EventMux())
eth.chainManager.SetProcessor(eth.blockProcessor)
eth.protocolManager = NewProtocolManager(config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.chainManager, chainDb)
if eth.protocolManager, err = NewProtocolManager(config.Mode, config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.chainManager, chainDb); err != nil {
return nil, err
}
eth.miner = miner.New(eth, eth.EventMux(), eth.pow)
eth.miner.SetGasPrice(config.GasPrice)
eth.miner.SetExtra(config.ExtraData)
@ -441,7 +443,7 @@ func (s *Ethereum) NodeInfo() *NodeInfo {
DiscPort: int(node.UDP),
TCPPort: int(node.TCP),
ListenAddr: s.net.ListenAddr,
Td: s.ChainManager().Td().String(),
Td: s.ChainManager().GetTd(s.ChainManager().CurrentBlock().Hash()).String(),
}
}

678
eth/downloader/downloader.go
View file

File diff suppressed because it is too large Load diff

760
eth/downloader/downloader_test.go
View file

File diff suppressed because it is too large Load diff

5
eth/downloader/metrics.go
View file

@ -42,4 +42,9 @@ var (
bodyReqTimer = metrics.NewTimer("eth/downloader/bodies/req")
bodyDropMeter = metrics.NewMeter("eth/downloader/bodies/drop")
bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout")
receiptInMeter = metrics.NewMeter("eth/downloader/receipts/in")
receiptReqTimer = metrics.NewTimer("eth/downloader/receipts/req")
receiptDropMeter = metrics.NewMeter("eth/downloader/receipts/drop")
receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout")
)

26
eth/downloader/modes.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package downloader
// SyncMode represents the synchronisation mode of the downloader.
type SyncMode int
const (
FullSync SyncMode = iota // Synchronise the entire block-chain history from full blocks
FastSync // Quikcly download the headers, full sync only at the chain head
LightSync // Download only the headers and terminate afterwards
)

200
eth/downloader/peer.go
View file

@ -36,10 +36,11 @@ type relativeHashFetcherFn func(common.Hash) error
type absoluteHashFetcherFn func(uint64, int) error
type blockFetcherFn func([]common.Hash) error
// Block header and body fethers belonging to eth/62 and above
// Block header and body fetchers belonging to eth/62 and above
type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error
type absoluteHeaderFetcherFn func(uint64, int, int, bool) error
type blockBodyFetcherFn func([]common.Hash) error
type receiptFetcherFn func([]common.Hash) error
var (
errAlreadyFetching = errors.New("already fetching blocks from peer")
@ -52,11 +53,14 @@ type peer struct {
id string // Unique identifier of the peer
head common.Hash // Hash of the peers latest known block
idle int32 // Current activity state of the peer (idle = 0, active = 1)
rep int32 // Simple peer reputation
blockIdle int32 // Current block activity state of the peer (idle = 0, active = 1)
receiptIdle int32 // Current receipt activity state of the peer (idle = 0, active = 1)
rep int32 // Simple peer reputation
capacity int32 // Number of blocks allowed to fetch per request
started time.Time // Time instance when the last fetch was started
blockCapacity int32 // Number of blocks (bodies) allowed to fetch per request
receiptCapacity int32 // Number of receipts allowed to fetch per request
blockStarted time.Time // Time instance when the last block (body)fetch was started
receiptStarted time.Time // Time instance when the last receipt fetch was started
ignored *set.Set // Set of hashes not to request (didn't have previously)
@ -68,6 +72,8 @@ type peer struct {
getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position
getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies
getReceipts receiptFetcherFn // [eth/63] Method to retrieve a batch of block transaction receipts
version int // Eth protocol version number to switch strategies
}
@ -75,12 +81,14 @@ type peer struct {
// mechanisms.
func newPeer(id string, version int, head common.Hash,
getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading
getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn) *peer {
getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn,
getReceipts receiptFetcherFn) *peer {
return &peer{
id: id,
head: head,
capacity: 1,
ignored: set.New(),
id: id,
head: head,
blockCapacity: 1,
receiptCapacity: 1,
ignored: set.New(),
getRelHashes: getRelHashes,
getAbsHashes: getAbsHashes,
@ -90,24 +98,28 @@ func newPeer(id string, version int, head common.Hash,
getAbsHeaders: getAbsHeaders,
getBlockBodies: getBlockBodies,
getReceipts: getReceipts,
version: version,
}
}
// Reset clears the internal state of a peer entity.
func (p *peer) Reset() {
atomic.StoreInt32(&p.idle, 0)
atomic.StoreInt32(&p.capacity, 1)
atomic.StoreInt32(&p.blockIdle, 0)
atomic.StoreInt32(&p.receiptIdle, 0)
atomic.StoreInt32(&p.blockCapacity, 1)
atomic.StoreInt32(&p.receiptCapacity, 1)
p.ignored.Clear()
}
// Fetch61 sends a block retrieval request to the remote peer.
func (p *peer) Fetch61(request *fetchRequest) error {
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
}
p.started = time.Now()
p.blockStarted = time.Now()
// Convert the hash set to a retrievable slice
hashes := make([]common.Hash, 0, len(request.Hashes))
@ -119,13 +131,13 @@ func (p *peer) Fetch61(request *fetchRequest) error {
return nil
}
// Fetch sends a block body retrieval request to the remote peer.
func (p *peer) Fetch(request *fetchRequest) error {
// FetchBodies sends a block body retrieval request to the remote peer.
func (p *peer) FetchBodies(request *fetchRequest) error {
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
}
p.started = time.Now()
p.blockStarted = time.Now()
// Convert the header set to a retrievable slice
hashes := make([]common.Hash, 0, len(request.Headers))
@ -137,55 +149,64 @@ func (p *peer) Fetch(request *fetchRequest) error {
return nil
}
// SetIdle61 sets the peer to idle, allowing it to execute new retrieval requests.
// FetchReceipts sends a receipt retrieval request to the remote peer.
func (p *peer) FetchReceipts(request *fetchRequest) error {
// Short circuit if the peer is already fetching
if !atomic.CompareAndSwapInt32(&p.receiptIdle, 0, 1) {
return errAlreadyFetching
}
p.receiptStarted = time.Now()
// Convert the header set to a retrievable slice
hashes := make([]common.Hash, 0, len(request.Headers))
for _, header := range request.Headers {
hashes = append(hashes, header.Hash())
}
go p.getReceipts(hashes)
return nil
}
// SetBlocksIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block retrieval allowance will also be updated either up- or downwards,
// depending on whether the previous fetch completed in time or not.
func (p *peer) SetIdle61() {
// Update the peer's download allowance based on previous performance
scale := 2.0
if time.Since(p.started) > blockSoftTTL {
scale = 0.5
if time.Since(p.started) > blockHardTTL {
scale = 1 / float64(MaxBlockFetch) // reduces capacity to 1
}
}
for {
// Calculate the new download bandwidth allowance
prev := atomic.LoadInt32(&p.capacity)
next := int32(math.Max(1, math.Min(float64(MaxBlockFetch), float64(prev)*scale)))
// Try to update the old value
if atomic.CompareAndSwapInt32(&p.capacity, prev, next) {
// If we're having problems at 1 capacity, try to find better peers
if next == 1 {
p.Demote()
}
break
}
}
// Set the peer to idle to allow further block requests
atomic.StoreInt32(&p.idle, 0)
func (p *peer) SetBlocksIdle() {
p.setIdle(p.blockStarted, blockSoftTTL, blockHardTTL, MaxBlockFetch, &p.blockCapacity, &p.blockIdle)
}
// SetIdle sets the peer to idle, allowing it to execute new retrieval requests.
// SetBodiesIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block body retrieval allowance will also be updated either up- or downwards,
// depending on whether the previous fetch completed in time or not.
func (p *peer) SetIdle() {
func (p *peer) SetBodiesIdle() {
p.setIdle(p.blockStarted, bodySoftTTL, bodyHardTTL, MaxBlockFetch, &p.blockCapacity, &p.blockIdle)
}
// SetReceiptsIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its receipt retrieval allowance will also be updated either up- or downwards,
// depending on whether the previous fetch completed in time or not.
func (p *peer) SetReceiptsIdle() {
p.setIdle(p.receiptStarted, receiptSoftTTL, receiptHardTTL, MaxReceiptFetch, &p.receiptCapacity, &p.receiptIdle)
}
// setIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its data retrieval allowance will also be updated either up- or downwards,
// depending on whether the previous fetch completed in time or not.
func (p *peer) setIdle(started time.Time, softTTL, hardTTL time.Duration, maxFetch int, capacity, idle *int32) {
// Update the peer's download allowance based on previous performance
scale := 2.0
if time.Since(p.started) > bodySoftTTL {
if time.Since(started) > softTTL {
scale = 0.5
if time.Since(p.started) > bodyHardTTL {
scale = 1 / float64(MaxBodyFetch) // reduces capacity to 1
if time.Since(started) > hardTTL {
scale = 1 / float64(maxFetch) // reduces capacity to 1
}
}
for {
// Calculate the new download bandwidth allowance
prev := atomic.LoadInt32(&p.capacity)
next := int32(math.Max(1, math.Min(float64(MaxBodyFetch), float64(prev)*scale)))
prev := atomic.LoadInt32(capacity)
next := int32(math.Max(1, math.Min(float64(maxFetch), float64(prev)*scale)))
// Try to update the old value
if atomic.CompareAndSwapInt32(&p.capacity, prev, next) {
if atomic.CompareAndSwapInt32(capacity, prev, next) {
// If we're having problems at 1 capacity, try to find better peers
if next == 1 {
p.Demote()
@ -193,14 +214,20 @@ func (p *peer) SetIdle() {
break
}
}
// Set the peer to idle to allow further block requests
atomic.StoreInt32(&p.idle, 0)
// Set the peer to idle to allow further fetch requests
atomic.StoreInt32(idle, 0)
}
// Capacity retrieves the peers block download allowance based on its previously
// discovered bandwidth capacity.
func (p *peer) Capacity() int {
return int(atomic.LoadInt32(&p.capacity))
// BlockCapacity retrieves the peers block download allowance based on its
// previously discovered bandwidth capacity.
func (p *peer) BlockCapacity() int {
return int(atomic.LoadInt32(&p.blockCapacity))
}
// ReceiptCapacity retrieves the peers block download allowance based on its
// previously discovered bandwidth capacity.
func (p *peer) ReceiptCapacity() int {
return int(atomic.LoadInt32(&p.receiptCapacity))
}
// Promote increases the peer's reputation.
@ -226,7 +253,8 @@ func (p *peer) Demote() {
func (p *peer) String() string {
return fmt.Sprintf("Peer %s [%s]", p.id,
fmt.Sprintf("reputation %3d, ", atomic.LoadInt32(&p.rep))+
fmt.Sprintf("capacity %3d, ", atomic.LoadInt32(&p.capacity))+
fmt.Sprintf("block cap %3d, ", atomic.LoadInt32(&p.blockCapacity))+
fmt.Sprintf("receipt cap %3d, ", atomic.LoadInt32(&p.receiptCapacity))+
fmt.Sprintf("ignored %4d", p.ignored.Size()),
)
}
@ -310,26 +338,52 @@ func (ps *peerSet) AllPeers() []*peer {
return list
}
// IdlePeers retrieves a flat list of all the currently idle peers within the
// BlockIdlePeers retrieves a flat list of all the currently idle peers within the
// active peer set, ordered by their reputation.
func (ps *peerSet) IdlePeers(version int) []*peer {
func (ps *peerSet) BlockIdlePeers(version int) ([]*peer, int) {
ps.lock.RLock()
defer ps.lock.RUnlock()
list := make([]*peer, 0, len(ps.peers))
idle, total := make([]*peer, 0, len(ps.peers)), 0
for _, p := range ps.peers {
if (version == eth61 && p.version == eth61) || (version >= eth62 && p.version >= eth62) {
if atomic.LoadInt32(&p.idle) == 0 {
list = append(list, p)
if (version == 61 && p.version == 61) || (version >= 62 && p.version >= 62) {
if atomic.LoadInt32(&p.blockIdle) == 0 {
idle = append(idle, p)
}
total++
}
}
for i := 0; i < len(idle); i++ {
for j := i + 1; j < len(idle); j++ {
if atomic.LoadInt32(&idle[i].rep) < atomic.LoadInt32(&idle[j].rep) {
idle[i], idle[j] = idle[j], idle[i]
}
}
}
for i := 0; i < len(list); i++ {
for j := i + 1; j < len(list); j++ {
if atomic.LoadInt32(&list[i].rep) < atomic.LoadInt32(&list[j].rep) {
list[i], list[j] = list[j], list[i]
}
}
}
return list
return idle, total
}
// ReceiptIdlePeers retrieves a flat list of all the currently idle peers within the
// active peer set, ordered by their reputation.
func (ps *peerSet) ReceiptIdlePeers() ([]*peer, int) {
ps.lock.RLock()
defer ps.lock.RUnlock()
idle, total := make([]*peer, 0, len(ps.peers)), 0
for _, p := range ps.peers {
if p.version >= 63 {
if atomic.LoadInt32(&p.receiptIdle) == 0 {
idle = append(idle, p)
}
total++
}
}
for i := 0; i < len(idle); i++ {
for j := i + 1; j < len(idle); j++ {
if atomic.LoadInt32(&idle[i].rep) < atomic.LoadInt32(&idle[j].rep) {
idle[i], idle[j] = idle[j], idle[i]
}
}
}
return idle, total
}

596
eth/downloader/queue.go
View file

@ -29,11 +29,12 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/rcrowley/go-metrics"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)
var (
blockCacheLimit = 8 * MaxBlockFetch // Maximum number of blocks to cache before throttling the download
blockCacheLimit = 1024 // Maximum number of blocks to cache before throttling the download
)
var (
@ -41,29 +42,47 @@ var (
errStaleDelivery = errors.New("stale delivery")
)
// fetchRequest is a currently running block retrieval operation.
// fetchRequest is a currently running data retrieval operation.
type fetchRequest struct {
Peer *peer // Peer to which the request was sent
Hashes map[common.Hash]int // [eth/61] Requested hashes with their insertion index (priority)
Hashes map[common.Hash]int // [eth/61] Requested block with their insertion index (priority)
Headers []*types.Header // [eth/62] Requested headers, sorted by request order
Time time.Time // Time when the request was made
}
// fetchResult is the assembly collecting partial results from potentially more
// than one fetcher routines, until all outstanding retrievals complete and the
// result as a whole can be processed.
type fetchResult struct {
Pending int // Number of data fetches still pending
Header *types.Header
Uncles []*types.Header
Transactions types.Transactions
Receipts types.Receipts
}
// queue represents hashes that are either need fetching or are being fetched
type queue struct {
hashPool map[common.Hash]int // [eth/61] Pending hashes, mapping to their insertion index (priority)
hashQueue *prque.Prque // [eth/61] Priority queue of the block hashes to fetch
hashCounter int // [eth/61] Counter indexing the added hashes to ensure retrieval order
headerPool map[common.Hash]*types.Header // [eth/62] Pending headers, mapping from their hashes
headerQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the bodies for
headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
pendPool map[string]*fetchRequest // Currently pending block retrieval operations
blockTaskPool map[common.Hash]*types.Header // [eth/62] Pending block (body) retrieval tasks, mapping hashes to headers
blockTaskQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the blocks (bodies) for
blockPendPool map[string]*fetchRequest // [eth/62] Currently pending block (body) retrieval operations
blockDonePool map[common.Hash]struct{} // [eth/62] Set of the completed block (body) fetches
blockPool map[common.Hash]uint64 // Hash-set of the downloaded data blocks, mapping to cache indexes
blockCache []*Block // Downloaded but not yet delivered blocks
blockOffset uint64 // Offset of the first cached block in the block-chain
receiptTaskPool map[common.Hash]*types.Header // [eth/63] Pending receipt retrieval tasks, mapping hashes to headers
receiptTaskQueue *prque.Prque // [eth/63] Priority queue of the headers to fetch the receipts for
receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations
receiptDonePool map[common.Hash]struct{} // [eth/63] Set of the completed receipt fetches
resultCache []*fetchResult // Downloaded but not yet delivered fetch results
resultOffset uint64 // Offset of the first cached fetch result in the block-chain
resultParts int // Number of fetch components required to complete an item
lock sync.RWMutex
}
@ -71,13 +90,17 @@ type queue struct {
// newQueue creates a new download queue for scheduling block retrieval.
func newQueue() *queue {
return &queue{
hashPool: make(map[common.Hash]int),
hashQueue: prque.New(),
headerPool: make(map[common.Hash]*types.Header),
headerQueue: prque.New(),
pendPool: make(map[string]*fetchRequest),
blockPool: make(map[common.Hash]uint64),
blockCache: make([]*Block, blockCacheLimit),
hashPool: make(map[common.Hash]int),
hashQueue: prque.New(),
blockTaskPool: make(map[common.Hash]*types.Header),
blockTaskQueue: prque.New(),
blockPendPool: make(map[string]*fetchRequest),
blockDonePool: make(map[common.Hash]struct{}),
receiptTaskPool: make(map[common.Hash]*types.Header),
receiptTaskQueue: prque.New(),
receiptPendPool: make(map[string]*fetchRequest),
receiptDonePool: make(map[common.Hash]struct{}),
resultCache: make([]*fetchResult, blockCacheLimit),
}
}
@ -90,32 +113,37 @@ func (q *queue) Reset() {
q.hashQueue.Reset()
q.hashCounter = 0
q.headerPool = make(map[common.Hash]*types.Header)
q.headerQueue.Reset()
q.headerHead = common.Hash{}
q.pendPool = make(map[string]*fetchRequest)
q.blockTaskPool = make(map[common.Hash]*types.Header)
q.blockTaskQueue.Reset()
q.blockPendPool = make(map[string]*fetchRequest)
q.blockDonePool = make(map[common.Hash]struct{})
q.blockPool = make(map[common.Hash]uint64)
q.blockOffset = 0
q.blockCache = make([]*Block, blockCacheLimit)
q.receiptTaskPool = make(map[common.Hash]*types.Header)
q.receiptTaskQueue.Reset()
q.receiptPendPool = make(map[string]*fetchRequest)
q.receiptDonePool = make(map[common.Hash]struct{})
q.resultCache = make([]*fetchResult, blockCacheLimit)
q.resultOffset = 0
q.resultParts = 0
}
// Size retrieves the number of blocks in the queue, returning separately for
// pending and already downloaded.
func (q *queue) Size() (int, int) {
// PendingBlocks retrieves the number of block (body) requests pending for retrieval.
func (q *queue) PendingBlocks() int {
q.lock.RLock()
defer q.lock.RUnlock()
return len(q.hashPool) + len(q.headerPool), len(q.blockPool)
return q.hashQueue.Size() + q.blockTaskQueue.Size()
}
// Pending retrieves the number of blocks pending for retrieval.
func (q *queue) Pending() int {
// PendingReceipts retrieves the number of block receipts pending for retrieval.
func (q *queue) PendingReceipts() int {
q.lock.RLock()
defer q.lock.RUnlock()
return q.hashQueue.Size() + q.headerQueue.Size()
return q.receiptTaskQueue.Size()
}
// InFlight retrieves the number of fetch requests currently in flight.
@ -123,44 +151,55 @@ func (q *queue) InFlight() int {
q.lock.RLock()
defer q.lock.RUnlock()
return len(q.pendPool)
return len(q.blockPendPool) + len(q.receiptPendPool)
}
// Throttle checks if the download should be throttled (active block fetches
// exceed block cache).
func (q *queue) Throttle() bool {
// Idle returns if the queue is fully idle or has some data still inside. This
// method is used by the tester to detect termination events.
func (q *queue) Idle() bool {
q.lock.RLock()
defer q.lock.RUnlock()
// Calculate the currently in-flight block requests
queued := q.hashQueue.Size() + q.blockTaskQueue.Size() + q.receiptTaskQueue.Size()
pending := len(q.blockPendPool) + len(q.receiptPendPool)
cached := len(q.blockDonePool) + len(q.receiptDonePool)
return (queued + pending + cached) == 0
}
// ThrottleBlocks checks if the download should be throttled (active block (body)
// fetches exceed block cache).
func (q *queue) ThrottleBlocks() bool {
q.lock.RLock()
defer q.lock.RUnlock()
// Calculate the currently in-flight block (body) requests
pending := 0
for _, request := range q.pendPool {
for _, request := range q.blockPendPool {
pending += len(request.Hashes) + len(request.Headers)
}
// Throttle if more blocks are in-flight than free space in the cache
return pending >= len(q.blockCache)-len(q.blockPool)
// Throttle if more blocks (bodies) are in-flight than free space in the cache
return pending >= len(q.resultCache)-len(q.blockDonePool)
}
// Has checks if a hash is within the download queue or not.
func (q *queue) Has(hash common.Hash) bool {
// ThrottleReceipts checks if the download should be throttled (active receipt
// fetches exceed block cache).
func (q *queue) ThrottleReceipts() bool {
q.lock.RLock()
defer q.lock.RUnlock()
if _, ok := q.hashPool[hash]; ok {
return true
// Calculate the currently in-flight receipt requests
pending := 0
for _, request := range q.receiptPendPool {
pending += len(request.Headers)
}
if _, ok := q.headerPool[hash]; ok {
return true
}
if _, ok := q.blockPool[hash]; ok {
return true
}
return false
// Throttle if more receipts are in-flight than free space in the cache
return pending >= len(q.resultCache)-len(q.receiptDonePool)
}
// Insert61 adds a set of hashes for the download queue for scheduling, returning
// Schedule61 adds a set of hashes for the download queue for scheduling, returning
// the new hashes encountered.
func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
func (q *queue) Schedule61(hashes []common.Hash, fifo bool) []common.Hash {
q.lock.Lock()
defer q.lock.Unlock()
@ -186,22 +225,17 @@ func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
return inserts
}
// Insert adds a set of headers for the download queue for scheduling, returning
// Schedule adds a set of headers for the download queue for scheduling, returning
// the new headers encountered.
func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header {
func (q *queue) Schedule(headers []*types.Header, from uint64, receipts bool) []*types.Header {
q.lock.Lock()
defer q.lock.Unlock()
// Insert all the headers prioritized by the contained block number
inserts := make([]*types.Header, 0, len(headers))
for _, header := range headers {
// Make sure no duplicate requests are executed
hash := header.Hash()
if _, ok := q.headerPool[hash]; ok {
glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled", header.Number.Uint64(), hash[:4])
continue
}
// Make sure chain order is honored and preserved throughout
hash := header.Hash()
if header.Number == nil || header.Number.Uint64() != from {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ordering, expected %d", header.Number, hash[:4], from)
break
@ -210,69 +244,72 @@ func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ancestry", header.Number, hash[:4])
break
}
// Queue the header for body retrieval
// Make sure no duplicate requests are executed
if _, ok := q.blockTaskPool[hash]; ok {
glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled for block fetch", header.Number.Uint64(), hash[:4])
continue
}
if _, ok := q.receiptTaskPool[hash]; ok {
glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled for receipt fetch", header.Number.Uint64(), hash[:4])
continue
}
// Queue the header for content retrieval
q.blockTaskPool[hash] = header
q.blockTaskQueue.Push(header, -float32(header.Number.Uint64()))
if receipts {
q.receiptTaskPool[hash] = header
q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64()))
}
inserts = append(inserts, header)
q.headerPool[hash] = header
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
q.headerHead = hash
from++
}
return inserts
}
// GetHeadBlock retrieves the first block from the cache, or nil if it hasn't
// GetHeadResult retrieves the first fetch result from the cache, or nil if it hasn't
// been downloaded yet (or simply non existent).
func (q *queue) GetHeadBlock() *Block {
func (q *queue) GetHeadResult() *fetchResult {
q.lock.RLock()
defer q.lock.RUnlock()
if len(q.blockCache) == 0 {
if len(q.resultCache) == 0 || q.resultCache[0] == nil {
return nil
}
return q.blockCache[0]
}
// GetBlock retrieves a downloaded block, or nil if non-existent.
func (q *queue) GetBlock(hash common.Hash) *Block {
q.lock.RLock()
defer q.lock.RUnlock()
// Short circuit if the block hasn't been downloaded yet
index, ok := q.blockPool[hash]
if !ok {
if q.resultCache[0].Pending > 0 {
return nil
}
// Return the block if it's still available in the cache
if q.blockOffset <= index && index < q.blockOffset+uint64(len(q.blockCache)) {
return q.blockCache[index-q.blockOffset]
}
return nil
return q.resultCache[0]
}
// TakeBlocks retrieves and permanently removes a batch of blocks from the cache.
func (q *queue) TakeBlocks() []*Block {
// TakeResults retrieves and permanently removes a batch of fetch results from
// the cache.
func (q *queue) TakeResults() []*fetchResult {
q.lock.Lock()
defer q.lock.Unlock()
// Accumulate all available blocks
blocks := []*Block{}
for _, block := range q.blockCache {
if block == nil {
// Accumulate all available results
results := []*fetchResult{}
for _, result := range q.resultCache {
if result == nil || result.Pending > 0 {
break
}
blocks = append(blocks, block)
delete(q.blockPool, block.RawBlock.Hash())
}
// Delete the blocks from the slice and let them be garbage collected
// without this slice trick the blocks would stay in memory until nil
// would be assigned to q.blocks
copy(q.blockCache, q.blockCache[len(blocks):])
for k, n := len(q.blockCache)-len(blocks), len(q.blockCache); k < n; k++ {
q.blockCache[k] = nil
}
q.blockOffset += uint64(len(blocks))
results = append(results, result)
return blocks
hash := result.Header.Hash()
delete(q.blockDonePool, hash)
delete(q.receiptDonePool, hash)
}
// Delete the results from the slice and let them be garbage collected
// without this slice trick the results would stay in memory until nil
// would be assigned to them.
copy(q.resultCache, q.resultCache[len(results):])
for k, n := len(q.resultCache)-len(results), len(q.resultCache); k < n; k++ {
q.resultCache[k] = nil
}
q.resultOffset += uint64(len(results))
return results
}
// Reserve61 reserves a set of hashes for the given peer, skipping any previously
@ -286,12 +323,12 @@ func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
if q.hashQueue.Empty() {
return nil
}
if _, ok := q.pendPool[p.id]; ok {
if _, ok := q.blockPendPool[p.id]; ok {
return nil
}
// Calculate an upper limit on the hashes we might fetch (i.e. throttling)
space := len(q.blockCache) - len(q.blockPool)
for _, request := range q.pendPool {
space := len(q.resultCache) - len(q.blockDonePool)
for _, request := range q.blockPendPool {
space -= len(request.Hashes)
}
// Retrieve a batch of hashes, skipping previously failed ones
@ -319,49 +356,82 @@ func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
Hashes: send,
Time: time.Now(),
}
q.pendPool[p.id] = request
q.blockPendPool[p.id] = request
return request
}
// Reserve reserves a set of headers for the given peer, skipping any previously
// failed download. Beside the next batch of needed fetches, it also returns a
// flag whether empty blocks were queued requiring processing.
func (q *queue) Reserve(p *peer, count int) (*fetchRequest, bool, error) {
// ReserveBlocks reserves a set of body fetches for the given peer, skipping any
// previously failed downloads. Beside the next batch of needed fetches, it also
// returns a flag whether empty blocks were queued requiring processing.
func (q *queue) ReserveBlocks(p *peer, count int) (*fetchRequest, bool, error) {
noop := func(header *types.Header) bool {
return header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash
}
return q.reserveFetch(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, noop)
}
// ReserveReceipts reserves a set of receipt fetches for the given peer, skipping
// any previously failed downloads. Beside the next batch of needed fetches, it
// also returns a flag whether empty receipts were queued requiring importing.
func (q *queue) ReserveReceipts(p *peer, count int) (*fetchRequest, bool, error) {
noop := func(header *types.Header) bool {
return header.ReceiptHash == types.EmptyRootHash
}
return q.reserveFetch(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, noop)
}
// reserveFetch reserves a set of data download operations for a given peer,
// skipping any previously failed ones. This method is a generic version used
// by the individual special reservation functions.
func (q *queue) reserveFetch(p *peer, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque,
pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, noop func(*types.Header) bool) (*fetchRequest, bool, error) {
q.lock.Lock()
defer q.lock.Unlock()
// Short circuit if the pool has been depleted, or if the peer's already
// downloading something (sanity check not to corrupt state)
if q.headerQueue.Empty() {
if taskQueue.Empty() {
return nil, false, nil
}
if _, ok := q.pendPool[p.id]; ok {
if _, ok := pendPool[p.id]; ok {
return nil, false, nil
}
// Calculate an upper limit on the bodies we might fetch (i.e. throttling)
space := len(q.blockCache) - len(q.blockPool)
for _, request := range q.pendPool {
// Calculate an upper limit on the items we might fetch (i.e. throttling)
space := len(q.resultCache) - len(donePool)
for _, request := range pendPool {
space -= len(request.Headers)
}
// Retrieve a batch of headers, skipping previously failed ones
// Retrieve a batch of tasks, skipping previously failed ones
send := make([]*types.Header, 0, count)
skip := make([]*types.Header, 0)
process := false
for proc := 0; proc < space && len(send) < count && !q.headerQueue.Empty(); proc++ {
header := q.headerQueue.PopItem().(*types.Header)
progress := false
for proc := 0; proc < space && len(send) < count && !taskQueue.Empty(); proc++ {
header := taskQueue.PopItem().(*types.Header)
// If the header defines an empty block, deliver straight
if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
if err := q.enqueue("", types.NewBlockWithHeader(header)); err != nil {
return nil, false, errInvalidChain
// If we're the first to request this task, initialize the result container
index := int(header.Number.Int64() - int64(q.resultOffset))
if index >= len(q.resultCache) || index < 0 {
return nil, false, errInvalidChain
}
if q.resultCache[index] == nil {
q.resultCache[index] = &fetchResult{
Pending: q.resultParts,
Header: header,
}
delete(q.headerPool, header.Hash())
process, space, proc = true, space-1, proc-1
}
// If this fetch task is a noop, skip this fetch operation
if noop(header) {
donePool[header.Hash()] = struct{}{}
delete(taskPool, header.Hash())
space, proc = space-1, proc-1
q.resultCache[index].Pending--
progress = true
continue
}
// If it's a content block, add to the body fetch request
// Otherwise if not a known unknown block, add to the retrieve list
if p.ignored.Has(header.Hash()) {
skip = append(skip, header)
} else {
@ -370,24 +440,41 @@ func (q *queue) Reserve(p *peer, count int) (*fetchRequest, bool, error) {
}
// Merge all the skipped headers back
for _, header := range skip {
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
// Assemble and return the block download request
if len(send) == 0 {
return nil, process, nil
return nil, progress, nil
}
request := &fetchRequest{
Peer: p,
Headers: send,
Time: time.Now(),
}
q.pendPool[p.id] = request
pendPool[p.id] = request
return request, process, nil
return request, progress, nil
}
// Cancel aborts a fetch request, returning all pending hashes to the queue.
func (q *queue) Cancel(request *fetchRequest) {
// Cancel61 aborts a fetch request, returning all pending hashes to the queue.
func (q *queue) Cancel61(request *fetchRequest) {
q.cancel(request, nil, q.blockPendPool)
}
// CancelBlocks aborts a body fetch request, returning all pending hashes to the
// task queue.
func (q *queue) CancelBlocks(request *fetchRequest) {
q.cancel(request, q.blockTaskQueue, q.blockPendPool)
}
// CancelReceipts aborts a body fetch request, returning all pending hashes to
// the task queue.
func (q *queue) CancelReceipts(request *fetchRequest) {
q.cancel(request, q.receiptTaskQueue, q.receiptPendPool)
}
// Cancel aborts a fetch request, returning all pending hashes to the task queue.
func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest) {
q.lock.Lock()
defer q.lock.Unlock()
@ -395,20 +482,62 @@ func (q *queue) Cancel(request *fetchRequest) {
q.hashQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
delete(q.pendPool, request.Peer.id)
delete(pendPool, request.Peer.id)
}
// Expire checks for in flight requests that exceeded a timeout allowance,
// Revoke cancels all pending requests belonging to a given peer. This method is
// meant to be called during a peer drop to quickly reassign owned data fetches
// to remaining nodes.
func (q *queue) Revoke(peerId string) {
q.lock.Lock()
defer q.lock.Unlock()
if request, ok := q.blockPendPool[peerId]; ok {
for hash, index := range request.Hashes {
q.hashQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
q.blockTaskQueue.Push(header, -float32(header.Number.Uint64()))
}
delete(q.blockPendPool, peerId)
}
if request, ok := q.receiptPendPool[peerId]; ok {
for _, header := range request.Headers {
q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64()))
}
delete(q.receiptPendPool, peerId)
}
}
// Expire61 checks for in flight requests that exceeded a timeout allowance,
// canceling them and returning the responsible peers for penalization.
func (q *queue) Expire(timeout time.Duration) []string {
func (q *queue) Expire61(timeout time.Duration) []string {
return q.expire(timeout, q.blockPendPool, nil)
}
// ExpireBlocks checks for in flight block body requests that exceeded a timeout
// allowance, canceling them and returning the responsible peers for penalization.
func (q *queue) ExpireBlocks(timeout time.Duration) []string {
return q.expire(timeout, q.blockPendPool, q.blockTaskQueue)
}
// ExpireReceipts checks for in flight receipt requests that exceeded a timeout
// allowance, canceling them and returning the responsible peers for penalization.
func (q *queue) ExpireReceipts(timeout time.Duration) []string {
return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue)
}
// expire is the generic check that move expired tasks from a pending pool back
// into a task pool, returning all entities caught with expired tasks.
func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque) []string {
q.lock.Lock()
defer q.lock.Unlock()
// Iterate over the expired requests and return each to the queue
peers := []string{}
for id, request := range q.pendPool {
for id, request := range pendPool {
if time.Since(request.Time) > timeout {
// Update the metrics with the timeout
if len(request.Hashes) > 0 {
@ -421,14 +550,14 @@ func (q *queue) Expire(timeout time.Duration) []string {
q.hashQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
peers = append(peers, id)
}
}
// Remove the expired requests from the pending pool
for _, id := range peers {
delete(q.pendPool, id)
delete(pendPool, id)
}
return peers
}
@ -439,12 +568,12 @@ func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
defer q.lock.Unlock()
// Short circuit if the blocks were never requested
request := q.pendPool[id]
request := q.blockPendPool[id]
if request == nil {
return errNoFetchesPending
}
blockReqTimer.UpdateSince(request.Time)
delete(q.pendPool, id)
delete(q.blockPendPool, id)
// If no blocks were retrieved, mark them as unavailable for the origin peer
if len(blocks) == 0 {
@ -461,10 +590,19 @@ func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
errs = append(errs, fmt.Errorf("non-requested block %x", hash))
continue
}
// Queue the block up for processing
if err := q.enqueue(id, block); err != nil {
return err
// Reconstruct the next result if contents match up
index := int(block.Number().Int64() - int64(q.resultOffset))
if index >= len(q.resultCache) || index < 0 {
errs = []error{errInvalidChain}
break
}
q.resultCache[index] = &fetchResult{
Header: block.Header(),
Transactions: block.Transactions(),
Uncles: block.Uncles(),
}
q.blockDonePool[block.Hash()] = struct{}{}
delete(request.Hashes, hash)
delete(q.hashPool, hash)
}
@ -473,72 +611,12 @@ func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
q.hashQueue.Push(hash, float32(index))
}
// If none of the blocks were good, it's a stale delivery
if len(errs) != 0 {
if len(errs) == len(blocks) {
return errStaleDelivery
}
return fmt.Errorf("multiple failures: %v", errs)
}
return nil
}
// Deliver injects a block body retrieval response into the download queue.
func (q *queue) Deliver(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
q.lock.Lock()
defer q.lock.Unlock()
// Short circuit if the block bodies were never requested
request := q.pendPool[id]
if request == nil {
return errNoFetchesPending
}
bodyReqTimer.UpdateSince(request.Time)
delete(q.pendPool, id)
// If no block bodies were retrieved, mark them as unavailable for the origin peer
if len(txLists) == 0 || len(uncleLists) == 0 {
for hash, _ := range request.Headers {
request.Peer.ignored.Add(hash)
}
}
// Assemble each of the block bodies with their headers and queue for processing
errs := make([]error, 0)
for i, header := range request.Headers {
// Short circuit block assembly if no more bodies are found
if i >= len(txLists) || i >= len(uncleLists) {
break
}
// Reconstruct the next block if contents match up
if types.DeriveSha(types.Transactions(txLists[i])) != header.TxHash || types.CalcUncleHash(uncleLists[i]) != header.UncleHash {
errs = []error{errInvalidBody}
break
}
block := types.NewBlockWithHeader(header).WithBody(txLists[i], uncleLists[i])
// Queue the block up for processing
if err := q.enqueue(id, block); err != nil {
errs = []error{err}
break
}
request.Headers[i] = nil
delete(q.headerPool, header.Hash())
}
// Return all failed or missing fetches to the queue
for _, header := range request.Headers {
if header != nil {
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
}
}
// If none of the blocks were good, it's a stale delivery
switch {
case len(errs) == 0:
return nil
case len(errs) == 1 && errs[0] == errInvalidBody:
return errInvalidBody
case len(errs) == 1 && errs[0] == errInvalidChain:
return errInvalidChain
case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBlock):
return errs[0]
case len(errs) == len(request.Headers):
return errStaleDelivery
@ -548,29 +626,105 @@ func (q *queue) Deliver(id string, txLists [][]*types.Transaction, uncleLists []
}
}
// enqueue inserts a new block into the final delivery queue, waiting for pickup
// by the processor.
func (q *queue) enqueue(origin string, block *types.Block) error {
// If a requested block falls out of the range, the hash chain is invalid
index := int(int64(block.NumberU64()) - int64(q.blockOffset))
if index >= len(q.blockCache) || index < 0 {
return errInvalidChain
// DeliverBlocks injects a block (body) retrieval response into the results queue.
func (q *queue) DeliverBlocks(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Transactions(txLists[index])) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
return errInvalidBody
}
result.Transactions = txLists[index]
result.Uncles = uncleLists[index]
return nil
}
// Otherwise merge the block and mark the hash done
q.blockCache[index] = &Block{
RawBlock: block,
OriginPeer: origin,
}
q.blockPool[block.Header().Hash()] = block.NumberU64()
return nil
return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, bodyReqTimer, len(txLists), reconstruct)
}
// Prepare configures the block cache offset to allow accepting inbound blocks.
func (q *queue) Prepare(offset uint64) {
// DeliverReceipts injects a receipt retrieval response into the results queue.
func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) error {
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Receipts(receiptList[index])) != header.ReceiptHash {
return errInvalidReceipt
}
result.Receipts = receiptList[index]
return nil
}
return q.deliver(id, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, receiptReqTimer, len(receiptList), reconstruct)
}
// deliver injects a data retrieval response into the results queue.
func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest,
donePool map[common.Hash]struct{}, reqTimer metrics.Timer, results int, reconstruct func(header *types.Header, index int, result *fetchResult) error) error {
q.lock.Lock()
defer q.lock.Unlock()
if q.blockOffset < offset {
q.blockOffset = offset
// Short circuit if the data was never requested
request := pendPool[id]
if request == nil {
return errNoFetchesPending
}
reqTimer.UpdateSince(request.Time)
delete(pendPool, id)
// If no data items were retrieved, mark them as unavailable for the origin peer
if results == 0 {
for hash, _ := range request.Headers {
request.Peer.ignored.Add(hash)
}
}
// Assemble each of the results with their headers and retrieved data parts
errs := make([]error, 0)
for i, header := range request.Headers {
// Short circuit assembly if no more fetch results are found
if i >= results {
break
}
// Reconstruct the next result if contents match up
index := int(header.Number.Int64() - int64(q.resultOffset))
if index >= len(q.resultCache) || index < 0 || q.resultCache[index] == nil {
errs = []error{errInvalidChain}
break
}
if err := reconstruct(header, i, q.resultCache[index]); err != nil {
errs = []error{err}
break
}
donePool[header.Hash()] = struct{}{}
q.resultCache[index].Pending--
// Clean up a successful fetch
request.Headers[i] = nil
delete(taskPool, header.Hash())
}
// Return all failed or missing fetches to the queue
for _, header := range request.Headers {
if header != nil {
taskQueue.Push(header, -float32(header.Number.Uint64()))
}
}
// If none of the blocks were good, it's a stale delivery
switch {
case len(errs) == 0:
return nil
case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBody || errs[0] == errInvalidReceipt):
return errs[0]
case len(errs) == len(request.Headers):
return errStaleDelivery
default:
return fmt.Errorf("multiple failures: %v", errs)
}
}
// Prepare configures the result cache to allow accepting and caching inbound
// fetch results.
func (q *queue) Prepare(offset uint64, parts int) {
q.lock.Lock()
defer q.lock.Unlock()
if q.resultOffset < offset {
q.resultOffset = offset
}
q.resultParts = parts
}

2
eth/fetcher/fetcher_test.go
View file

@ -45,7 +45,7 @@ var (
// contains a transaction and every 5th an uncle to allow testing correct block
// reassembly.
func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
blocks := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
blocks, _ := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
block.SetCoinbase(common.Address{seed})
// If the block number is multiple of 3, send a bonus transaction to the miner

158
eth/handler.go
View file

@ -17,6 +17,7 @@
package eth
import (
"errors"
"fmt"
"math"
"math/big"
@ -42,6 +43,10 @@ const (
estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header
)
// errIncompatibleConfig is returned if the requested protocols and configs are
// not compatible (low protocol version restrictions and high requirements).
var errIncompatibleConfig = errors.New("incompatible configuration")
func errResp(code errCode, format string, v ...interface{}) error {
return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...))
}
@ -49,17 +54,8 @@ func errResp(code errCode, format string, v ...interface{}) error {
type hashFetcherFn func(common.Hash) error
type blockFetcherFn func([]common.Hash) error
// extProt is an interface which is passed around so we can expose GetHashes and GetBlock without exposing it to the rest of the protocol
// extProt is passed around to peers which require to GetHashes and GetBlocks
type extProt struct {
getHashes hashFetcherFn
getBlocks blockFetcherFn
}
func (ep extProt) GetHashes(hash common.Hash) error { return ep.getHashes(hash) }
func (ep extProt) GetBlock(hashes []common.Hash) error { return ep.getBlocks(hashes) }
type ProtocolManager struct {
mode Mode
txpool txPool
chainman *core.ChainManager
chaindb ethdb.Database
@ -87,9 +83,10 @@ type ProtocolManager struct {
// NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
// with the ethereum network.
func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, chainman *core.ChainManager, chaindb ethdb.Database) *ProtocolManager {
func NewProtocolManager(mode Mode, networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, chainman *core.ChainManager, chaindb ethdb.Database) (*ProtocolManager, error) {
// Create the protocol manager with the base fields
manager := &ProtocolManager{
mode: mode,
eventMux: mux,
txpool: txpool,
chainman: chainman,
@ -100,11 +97,15 @@ func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow po
quitSync: make(chan struct{}),
}
// Initiate a sub-protocol for every implemented version we can handle
manager.SubProtocols = make([]p2p.Protocol, len(ProtocolVersions))
for i := 0; i < len(manager.SubProtocols); i++ {
version := ProtocolVersions[i]
manager.SubProtocols[i] = p2p.Protocol{
manager.SubProtocols = make([]p2p.Protocol, 0, len(ProtocolVersions))
for i, version := range ProtocolVersions {
// Skip protocol version if incompatible with the mode of operation
if minimumProtocolVersion[mode] > version {
continue
}
// Compatible, initialize the sub-protocol
version := version // Closure for the run
manager.SubProtocols = append(manager.SubProtocols, p2p.Protocol{
Name: "eth",
Version: version,
Length: ProtocolLengths[i],
@ -113,20 +114,34 @@ func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow po
manager.newPeerCh <- peer
return manager.handle(peer)
},
}
})
}
if len(manager.SubProtocols) == 0 {
return nil, errIncompatibleConfig
}
// Construct the different synchronisation mechanisms
manager.downloader = downloader.New(manager.eventMux, manager.chainman.HasBlock, manager.chainman.GetBlock, manager.chainman.CurrentBlock, manager.chainman.GetTd, manager.chainman.InsertChain, manager.removePeer)
var syncMode downloader.SyncMode
switch mode {
case ArchiveMode:
syncMode = downloader.FullSync
case FullMode:
syncMode = downloader.FastSync
case LightMode:
syncMode = downloader.LightSync
}
manager.downloader = downloader.New(syncMode, manager.eventMux, chainman.HasHeader, chainman.HasBlock, chainman.GetHeader,
chainman.GetBlock, chainman.CurrentHeader, chainman.CurrentBlock, chainman.CurrentFastBlock, chainman.GetTd,
chainman.InsertHeaderChain, chainman.InsertChain, chainman.InsertReceiptChain, manager.removePeer)
validator := func(block *types.Block, parent *types.Block) error {
return core.ValidateHeader(pow, block.Header(), parent.Header(), true, false)
}
heighter := func() uint64 {
return manager.chainman.CurrentBlock().NumberU64()
return chainman.CurrentBlock().NumberU64()
}
manager.fetcher = fetcher.New(manager.chainman.GetBlock, validator, manager.BroadcastBlock, heighter, manager.chainman.InsertChain, manager.removePeer)
manager.fetcher = fetcher.New(chainman.GetBlock, validator, manager.BroadcastBlock, heighter, chainman.InsertChain, manager.removePeer)
return manager
return manager, nil
}
func (pm *ProtocolManager) removePeer(id string) {
@ -206,7 +221,7 @@ func (pm *ProtocolManager) handle(p *peer) error {
// Register the peer in the downloader. If the downloader considers it banned, we disconnect
if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(),
p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks,
p.RequestHeadersByHash, p.RequestHeadersByNumber, p.RequestBodies); err != nil {
p.RequestHeadersByHash, p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts); err != nil {
return err
}
// Propagate existing transactions. new transactions appearing
@ -424,28 +439,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
}
case p.version >= eth62 && msg.Code == BlockBodiesMsg:
// A batch of block bodies arrived to one of our previous requests
var request blockBodiesData
if err := msg.Decode(&request); err != nil {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Deliver them all to the downloader for queuing
trasactions := make([][]*types.Transaction, len(request))
uncles := make([][]*types.Header, len(request))
for i, body := range request {
trasactions[i] = body.Transactions
uncles[i] = body.Uncles
}
// Filter out any explicitly requested bodies, deliver the rest to the downloader
if trasactions, uncles := pm.fetcher.FilterBodies(trasactions, uncles, time.Now()); len(trasactions) > 0 || len(uncles) > 0 {
err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
if err != nil {
glog.V(logger.Debug).Infoln(err)
}
}
case p.version >= eth62 && msg.Code == GetBlockBodiesMsg:
// Decode the retrieval message
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
@ -473,6 +466,28 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
return p.SendBlockBodiesRLP(bodies)
case p.version >= eth62 && msg.Code == BlockBodiesMsg:
// A batch of block bodies arrived to one of our previous requests
var request blockBodiesData
if err := msg.Decode(&request); err != nil {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Deliver them all to the downloader for queuing
trasactions := make([][]*types.Transaction, len(request))
uncles := make([][]*types.Header, len(request))
for i, body := range request {
trasactions[i] = body.Transactions
uncles[i] = body.Uncles
}
// Filter out any explicitly requested bodies, deliver the rest to the downloader
if trasactions, uncles := pm.fetcher.FilterBodies(trasactions, uncles, time.Now()); len(trasactions) > 0 || len(uncles) > 0 {
err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
if err != nil {
glog.V(logger.Debug).Infoln(err)
}
}
case p.version >= eth63 && msg.Code == GetNodeDataMsg:
// Decode the retrieval message
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
@ -510,22 +525,42 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
var (
hash common.Hash
bytes int
receipts []*types.Receipt
receipts []rlp.RawValue
)
for bytes < softResponseLimit && len(receipts) < downloader.MaxReceiptsFetch {
// Retrieve the hash of the next transaction receipt
for bytes < softResponseLimit && len(receipts) < downloader.MaxReceiptFetch {
// Retrieve the hash of the next block
if err := msgStream.Decode(&hash); err == rlp.EOL {
break
} else if err != nil {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Retrieve the requested receipt, stopping if enough was found
if receipt := core.GetReceipt(pm.chaindb, hash); receipt != nil {
receipts = append(receipts, receipt)
bytes += len(receipt.RlpEncode())
// Retrieve the requested block's receipts, skipping if unknown to us
results := core.GetBlockReceipts(pm.chaindb, hash)
if results == nil {
if header := pm.chainman.GetHeader(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
continue
}
}
// If known, encode and queue for response packet
if encoded, err := rlp.EncodeToBytes(results); err != nil {
glog.V(logger.Error).Infof("failed to encode receipt: %v", err)
} else {
receipts = append(receipts, encoded)
bytes += len(encoded)
}
}
return p.SendReceipts(receipts)
return p.SendReceiptsRLP(receipts)
case p.version >= eth63 && msg.Code == ReceiptsMsg:
// A batch of receipts arrived to one of our previous requests
var receipts [][]*types.Receipt
if err := msg.Decode(&receipts); err != nil {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Deliver all to the downloader
if err := pm.downloader.DeliverReceipts(p.id, receipts); err != nil {
glog.V(logger.Debug).Infof("failed to deliver receipts: %v", err)
}
case msg.Code == NewBlockHashesMsg:
// Retrieve and deseralize the remote new block hashes notification
@ -585,15 +620,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
request.Block.ReceivedAt = msg.ReceivedAt
// Mark the block's arrival for whatever reason
_, chainHead, _ := pm.chainman.Status()
jsonlogger.LogJson(&logger.EthChainReceivedNewBlock{
BlockHash: request.Block.Hash().Hex(),
BlockNumber: request.Block.Number(),
ChainHeadHash: chainHead.Hex(),
BlockPrevHash: request.Block.ParentHash().Hex(),
RemoteId: p.ID().String(),
})
// Mark the peer as owning the block and schedule it for import
p.MarkBlock(request.Block.Hash())
p.SetHead(request.Block.Hash())
@ -603,7 +629,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
// Update the peers total difficulty if needed, schedule a download if gapped
if request.TD.Cmp(p.Td()) > 0 {
p.SetTd(request.TD)
if request.TD.Cmp(new(big.Int).Add(pm.chainman.Td(), request.Block.Difficulty())) > 0 {
td := pm.chainman.GetTd(pm.chainman.CurrentBlock().Hash())
if request.TD.Cmp(new(big.Int).Add(td, request.Block.Difficulty())) > 0 {
go pm.synchronise(p)
}
}
@ -620,12 +648,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "transaction %d is nil", i)
}
p.MarkTransaction(tx.Hash())
// Log it's arrival for later analysis
jsonlogger.LogJson(&logger.EthTxReceived{
TxHash: tx.Hash().Hex(),
RemoteId: p.ID().String(),
})
}
pm.txpool.AddTransactions(txs)

57
eth/handler_test.go
View file

@ -17,12 +17,42 @@ import (
"github.com/ethereum/go-ethereum/params"
)
// Tests that protocol versions and modes of operations are matched up properly.
func TestProtocolCompatibility(t *testing.T) {
// Define the compatibility chart
tests := []struct {
version uint
mode Mode
compatible bool
}{
{61, ArchiveMode, true}, {62, ArchiveMode, true}, {63, ArchiveMode, true}, {64, ArchiveMode, true},
{61, FullMode, false}, {62, FullMode, false}, {63, FullMode, true}, {64, FullMode, true},
{61, LightMode, false}, {62, LightMode, false}, {63, LightMode, false}, {64, LightMode, true},
}
// Make sure anything we screw up is restored
backup := ProtocolVersions
defer func() { ProtocolVersions = backup }()
// Try all available compatibility configs and check for errors
for i, tt := range tests {
ProtocolVersions = []uint{tt.version}
pm, err := newTestProtocolManager(tt.mode, 0, nil, nil)
if pm != nil {
defer pm.Stop()
}
if (err == nil && !tt.compatible) || (err != nil && tt.compatible) {
t.Errorf("test %d: compatibility mismatch: have error %v, want compatibility %v", i, err, tt.compatible)
}
}
}
// Tests that hashes can be retrieved from a remote chain by hashes in reverse
// order.
func TestGetBlockHashes61(t *testing.T) { testGetBlockHashes(t, 61) }
func testGetBlockHashes(t *testing.T, protocol int) {
pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -65,7 +95,7 @@ func testGetBlockHashes(t *testing.T, protocol int) {
func TestGetBlockHashesFromNumber61(t *testing.T) { testGetBlockHashesFromNumber(t, 61) }
func testGetBlockHashesFromNumber(t *testing.T, protocol int) {
pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -105,7 +135,7 @@ func testGetBlockHashesFromNumber(t *testing.T, protocol int) {
func TestGetBlocks61(t *testing.T) { testGetBlocks(t, 61) }
func testGetBlocks(t *testing.T, protocol int) {
pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -177,7 +207,7 @@ func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) }
func TestGetBlockHeaders64(t *testing.T) { testGetBlockHeaders(t, 64) }
func testGetBlockHeaders(t *testing.T, protocol int) {
pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -303,7 +333,7 @@ func TestGetBlockBodies63(t *testing.T) { testGetBlockBodies(t, 63) }
func TestGetBlockBodies64(t *testing.T) { testGetBlockBodies(t, 64) }
func testGetBlockBodies(t *testing.T, protocol int) {
pm := newTestProtocolManager(downloader.MaxBlockFetch+15, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, downloader.MaxBlockFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -410,7 +440,7 @@ func testGetNodeData(t *testing.T, protocol int) {
}
}
// Assemble the test environment
pm := newTestProtocolManager(4, generator, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@ -499,20 +529,17 @@ func testGetReceipt(t *testing.T, protocol int) {
}
}
// Assemble the test environment
pm := newTestProtocolManager(4, generator, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
// Collect the hashes to request, and the response to expect
hashes := []common.Hash{}
hashes, receipts := []common.Hash{}, []types.Receipts{}
for i := uint64(0); i <= pm.chainman.CurrentBlock().NumberU64(); i++ {
for _, tx := range pm.chainman.GetBlockByNumber(i).Transactions() {
hashes = append(hashes, tx.Hash())
}
}
receipts := make([]*types.Receipt, len(hashes))
for i, hash := range hashes {
receipts[i] = core.GetReceipt(pm.chaindb, hash)
block := pm.chainman.GetBlockByNumber(i)
hashes = append(hashes, block.Hash())
receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash()))
}
// Send the hash request and verify the response
p2p.Send(peer.app, 0x0f, hashes)

22
eth/helper_test.go
View file

@ -28,7 +28,7 @@ var (
// newTestProtocolManager creates a new protocol manager for testing purposes,
// with the given number of blocks already known, and potential notification
// channels for different events.
func newTestProtocolManager(blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) *ProtocolManager {
func newTestProtocolManager(mode Mode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, error) {
var (
evmux = new(event.TypeMux)
pow = new(core.FakePow)
@ -38,11 +38,27 @@ func newTestProtocolManager(blocks int, generator func(int, *core.BlockGen), new
blockproc = core.NewBlockProcessor(db, pow, chainman, evmux)
)
chainman.SetProcessor(blockproc)
if _, err := chainman.InsertChain(core.GenerateChain(genesis, db, blocks, generator)); err != nil {
chain, _ := core.GenerateChain(genesis, db, blocks, generator)
if _, err := chainman.InsertChain(chain); err != nil {
panic(err)
}
pm := NewProtocolManager(NetworkId, evmux, &testTxPool{added: newtx}, pow, chainman, db)
pm, err := NewProtocolManager(mode, NetworkId, evmux, &testTxPool{added: newtx}, pow, chainman, db)
if err != nil {
return nil, err
}
pm.Start()
return pm, nil
}
// newTestProtocolManagerMust creates a new protocol manager for testing purposes,
// with the given number of blocks already known, and potential notification
// channels for different events. In case of an error, the constructor force-
// fails the test.
func newTestProtocolManagerMust(t *testing.T, mode Mode, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) *ProtocolManager {
pm, err := newTestProtocolManager(mode, blocks, generator, newtx)
if err != nil {
t.Fatalf("Failed to create protocol manager: %v", err)
}
return pm
}

6
eth/peer.go
View file

@ -197,9 +197,9 @@ func (p *peer) SendNodeData(data [][]byte) error {
return p2p.Send(p.rw, NodeDataMsg, data)
}
// SendReceipts sends a batch of transaction receipts, corresponding to the ones
// requested.
func (p *peer) SendReceipts(receipts []*types.Receipt) error {
// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the
// ones requested from an already RLP encoded format.
func (p *peer) SendReceiptsRLP(receipts []rlp.RawValue) error {
return p2p.Send(p.rw, ReceiptsMsg, receipts)
}

19
eth/protocol.go
View file

@ -26,6 +26,15 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
// Mode represents the mode of operation of the eth client.
type Mode int
const (
ArchiveMode Mode = iota // Maintain the entire blockchain history
FullMode // Maintain only a recent view of the blockchain
LightMode // Don't maintain any history, rather fetch on demand
)
// Constants to match up protocol versions and messages
const (
eth61 = 61
@ -34,11 +43,19 @@ const (
eth64 = 64
)
// minimumProtocolVersion is the minimum version of the protocol eth must run to
// support the desired mode of operation.
var minimumProtocolVersion = map[Mode]uint{
ArchiveMode: eth61,
FullMode: eth63,
LightMode: eth64,
}
// Supported versions of the eth protocol (first is primary).
var ProtocolVersions = []uint{eth64, eth63, eth62, eth61}
// Number of implemented message corresponding to different protocol versions.
var ProtocolLengths = []uint64{15, 12, 8, 9}
var ProtocolLengths = []uint64{19, 17, 8, 9}
const (
NetworkId = 1

6
eth/protocol_test.go
View file

@ -44,7 +44,7 @@ func TestStatusMsgErrors63(t *testing.T) { testStatusMsgErrors(t, 63) }
func TestStatusMsgErrors64(t *testing.T) { testStatusMsgErrors(t, 64) }
func testStatusMsgErrors(t *testing.T, protocol int) {
pm := newTestProtocolManager(0, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, 0, nil, nil)
td, currentBlock, genesis := pm.chainman.Status()
defer pm.Stop()
@ -99,7 +99,7 @@ func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) }
func testRecvTransactions(t *testing.T, protocol int) {
txAdded := make(chan []*types.Transaction)
pm := newTestProtocolManager(0, nil, txAdded)
pm := newTestProtocolManagerMust(t, ArchiveMode, 0, nil, txAdded)
p, _ := newTestPeer("peer", protocol, pm, true)
defer pm.Stop()
defer p.close()
@ -127,7 +127,7 @@ func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) }
func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) }
func testSendTransactions(t *testing.T, protocol int) {
pm := newTestProtocolManager(0, nil, nil)
pm := newTestProtocolManagerMust(t, ArchiveMode, 0, nil, nil)
defer pm.Stop()
// Fill the pool with big transactions.

3
eth/sync.go
View file

@ -160,7 +160,8 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
return
}
// Make sure the peer's TD is higher than our own. If not drop.
if peer.Td().Cmp(pm.chainman.Td()) <= 0 {
td := pm.chainman.GetTd(pm.chainman.CurrentBlock().Hash())
if peer.Td().Cmp(td) <= 0 {
return
}
// Otherwise try to sync with the downloader

2
miner/worker.go
View file

@ -305,7 +305,7 @@ func (self *worker) wait() {
self.mux.Post(core.ChainHeadEvent{block})
self.mux.Post(logs)
}
if err := core.PutBlockReceipts(self.chainDb, block, receipts); err != nil {
if err := core.PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
glog.V(logger.Warn).Infoln("error writing block receipts:", err)
}
}(block, work.state.Logs(), work.receipts)

8
rpc/api/debug.go
View file

@ -146,13 +146,7 @@ func (self *debugApi) SetHead(req *shared.Request) (interface{}, error) {
if err := self.codec.Decode(req.Params, &args); err != nil {
return nil, shared.NewDecodeParamError(err.Error())
}
block := self.xeth.EthBlockByNumber(args.BlockNumber)
if block == nil {
return nil, fmt.Errorf("block #%d not found", args.BlockNumber)
}
self.ethereum.ChainManager().SetHead(block)
self.ethereum.ChainManager().SetHead(uint64(args.BlockNumber))
return nil, nil
}

2
rpc/api/eth_args.go
View file

@ -838,7 +838,7 @@ func NewLogRes(log *state.Log) LogRes {
}
l.Address = newHexData(log.Address)
l.Data = newHexData(log.Data)
l.BlockNumber = newHexNum(log.Number)
l.BlockNumber = newHexNum(log.BlockNumber)
l.LogIndex = newHexNum(log.Index)
l.TransactionHash = newHexData(log.TxHash)
l.TransactionIndex = newHexNum(log.TxIndex)

4
rpc/api/parsing.go
View file

@ -453,8 +453,8 @@ func NewReceiptRes(rec *types.Receipt) *ReceiptRes {
v.ContractAddress = newHexData(rec.ContractAddress)
}
logs := make([]interface{}, len(rec.Logs()))
for i, log := range rec.Logs() {
logs := make([]interface{}, len(rec.Logs))
for i, log := range rec.Logs {
logs[i] = NewLogRes(log)
}
v.Logs = &logs