forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-20 21:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix test download and recover some download config & test

Browse source
This commit is contained in:
parmarrushabh 2018-10-31 18:46:57 +05:30
parent 2e7323fdd1
commit 6adb7c1d68
2 changed files with 211 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
eth/downloader/downloader.go
View file

@ -25,7 +25,7 @@ import (
"sync/atomic"
"time"
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
@ -38,8 +38,8 @@ import (
var (
MaxHashFetch = 512 // Amount of hashes to be fetched per retrieval request
MaxBlockFetch = 900 // Amount of blocks to be fetched per retrieval request
MaxHeaderFetch = 900 // Amount of block headers to be fetched per retrieval request
MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request
MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request
MaxSkeletonSize = 128 // Number of header fetches to need for a skeleton assembly
MaxBodyFetch = 128 // Amount of block bodies to be fetched per retrieval request
MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request
@ -56,8 +56,8 @@ var (
qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence
qosTuningImpact = 0.25 // Impact that a new tuning target has on the previous value
maxQueuedHeaders = 900 // [eth/62] Maximum number of headers to queue for import (DOS protection)
maxHeadersProcess = 900 // Number of header download results to import at once into the chain
maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection)
maxHeadersProcess = 2048 // Number of header download results to import at once into the chain
maxResultsProcess = 2048 // Number of content download results to import at once into the chain
fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
@ -172,6 +172,8 @@ type LightChain interface {
// BlockChain encapsulates functions required to sync a (full or fast) blockchain.
type BlockChain interface {
Config() *params.ChainConfig
UpdateM1()
LightChain
// HasBlock verifies a block's presence in the local chain.
@ -1322,11 +1324,47 @@ func (d *Downloader) processFullSyncContent() error {
if len(results) == 0 {
return nil
}
if d.chainInsertHook != nil {
d.chainInsertHook(results)
}
if err := d.importBlockResults(results); err != nil {
return err
if d.blockchain.Config() != nil && d.blockchain.Config().XDPoS != nil {
epoch := d.blockchain.Config().XDPoS.Epoch
gap := d.blockchain.Config().XDPoS.Gap
length := len(results)
start := int(results[0].Header.Number.Uint64() % epoch)
end := int(epoch - gap - uint64(start))
if end < 0 {
end = end + int(epoch)
}
start = 0
for {
if end >= length {
end = length - 1
}
inserts := make([]*fetchResult, end-start+1)
copy(inserts, results[start:end+1])
if len(inserts) > 0 {
if d.chainInsertHook != nil {
d.chainInsertHook(inserts)
}
if err := d.importBlockResults(inserts); err != nil {
return err
}
// prepare set of masternodes for the next epoch
if (inserts[len(inserts)-1].Header.Number.Uint64() % epoch) == (epoch - gap) {
d.blockchain.UpdateM1()
}
}
start = end + 1
end = end + int(epoch)
if start >= length {
break
}
}
} else {
if d.chainInsertHook != nil {
d.chainInsertHook(results)
}
if err := d.importBlockResults(results); err != nil {
return err
}
}
}
}
@ -1355,6 +1393,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error {
log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err)
return errInvalidChain
}
return nil
}
@ -1632,4 +1671,4 @@ func (d *Downloader) requestTTL() time.Duration {
ttl = ttlLimit
}
return ttl
}
}

318
eth/downloader/downloader_test.go
View file

@ -456,6 +456,10 @@ func (dl *downloadTester) dropPeer(id string) {
dl.downloader.UnregisterPeer(id)
}
// Config retrieves the blockchain's chain configuration.
func (dl *downloadTester) Config() *params.ChainConfig { return params.TestChainConfig }
func (dl *downloadTester) UpdateM1() {}
type downloadTesterPeer struct {
dl *downloadTester
id string
@ -1186,92 +1190,92 @@ func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
// Tests that upon detecting an invalid header, the recent ones are rolled back
// for various failure scenarios. Afterwards a full sync is attempted to make
// sure no state was corrupted.
//func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback(t, 63, FastSync) }
//func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) }
//func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(t, 64, LightSync) }
//
//func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
// t.Parallel()
//
// tester := newTester()
// defer tester.terminate()
//
// // Create a small enough block chain to download
// targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks
// hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
//
// // Attempt to sync with an attacker that feeds junk during the fast sync phase.
// // This should result in the last fsHeaderSafetyNet headers being rolled back.
// tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts)
// missing := fsHeaderSafetyNet + MaxHeaderFetch + 1
// delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing])
//
// if err := tester.sync("fast-attack", nil, mode); err == nil {
// t.Fatalf("succeeded fast attacker synchronisation")
// }
// if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch {
// t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
// }
// // Attempt to sync with an attacker that feeds junk during the block import phase.
// // This should result in both the last fsHeaderSafetyNet number of headers being
// // rolled back, and also the pivot point being reverted to a non-block status.
// tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts)
// missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
// delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in
// delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing])
//
// if err := tester.sync("block-attack", nil, mode); err == nil {
// t.Fatalf("succeeded block attacker synchronisation")
// }
// if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
// t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
// }
// if mode == FastSync {
// if head := tester.CurrentBlock().NumberU64(); head != 0 {
// t.Errorf("fast sync pivot block #%d not rolled back", head)
// }
// }
// // Attempt to sync with an attacker that withholds promised blocks after the
// // fast sync pivot point. This could be a trial to leave the node with a bad
// // but already imported pivot block.
// tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
// missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
//
// tester.downloader.syncInitHook = func(uint64, uint64) {
// for i := missing; i <= len(hashes); i++ {
// delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
// }
// tester.downloader.syncInitHook = nil
// }
//
// if err := tester.sync("withhold-attack", nil, mode); err == nil {
// t.Fatalf("succeeded withholding attacker synchronisation")
// }
// if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
// t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
// }
// if mode == FastSync {
// if head := tester.CurrentBlock().NumberU64(); head != 0 {
// t.Errorf("fast sync pivot block #%d not rolled back", head)
// }
// }
// // Synchronise with the valid peer and make sure sync succeeds. Since the last
// // rollback should also disable fast syncing for this process, verify that we
// // did a fresh full sync. Note, we can't assert anything about the receipts
// // since we won't purge the database of them, hence we can't use assertOwnChain.
// tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
// if err := tester.sync("valid", nil, mode); err != nil {
// t.Fatalf("failed to synchronise blocks: %v", err)
// }
// if hs := len(tester.ownHeaders); hs != len(headers) {
// t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers))
// }
// if mode != LightSync {
// if bs := len(tester.ownBlocks); bs != len(blocks) {
// t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks))
// }
// }
//}
func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback(t, 63, FastSync) }
func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) }
func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(t, 64, LightSync) }
func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
t.Parallel()
tester := newTester()
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Attempt to sync with an attacker that feeds junk during the fast sync phase.
// This should result in the last fsHeaderSafetyNet headers being rolled back.
tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts)
missing := fsHeaderSafetyNet + MaxHeaderFetch + 1
delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing])
if err := tester.sync("fast-attack", nil, mode); err == nil {
t.Fatalf("succeeded fast attacker synchronisation")
}
if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch {
t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
}
// Attempt to sync with an attacker that feeds junk during the block import phase.
// This should result in both the last fsHeaderSafetyNet number of headers being
// rolled back, and also the pivot point being reverted to a non-block status.
tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts)
missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing]) // Make sure the fast-attacker doesn't fill in
delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing])
if err := tester.sync("block-attack", nil, mode); err == nil {
t.Fatalf("succeeded block attacker synchronisation")
}
if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
}
if mode == FastSync {
if head := tester.CurrentBlock().NumberU64(); head != 0 {
t.Errorf("fast sync pivot block #%d not rolled back", head)
}
}
// Attempt to sync with an attacker that withholds promised blocks after the
// fast sync pivot point. This could be a trial to leave the node with a bad
// but already imported pivot block.
tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
tester.downloader.syncInitHook = func(uint64, uint64) {
for i := missing; i <= len(hashes); i++ {
delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
}
tester.downloader.syncInitHook = nil
}
if err := tester.sync("withhold-attack", nil, mode); err == nil {
t.Fatalf("succeeded withholding attacker synchronisation")
}
if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
}
if mode == FastSync {
if head := tester.CurrentBlock().NumberU64(); head != 0 {
t.Errorf("fast sync pivot block #%d not rolled back", head)
}
}
// Synchronise with the valid peer and make sure sync succeeds. Since the last
// rollback should also disable fast syncing for this process, verify that we
// did a fresh full sync. Note, we can't assert anything about the receipts
// since we won't purge the database of them, hence we can't use assertOwnChain.
tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
if hs := len(tester.ownHeaders); hs != len(headers) {
t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers))
}
if mode != LightSync {
if bs := len(tester.ownBlocks); bs != len(blocks) {
t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks))
}
}
}
// Tests that a peer advertising an high TD doesn't get to stall the downloader
// afterwards by not sending any useful hashes.
@ -1353,77 +1357,77 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
}
}
// Tests that synchronisation progress (origin block number, current block number
// and highest block number) is tracked and updated correctly.
//func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) }
//func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) }
//func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) }
//func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) }
//func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) }
//func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) }
//
//func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// t.Parallel()
//
// tester := newTester()
// defer tester.terminate()
//
// // Create a small enough block chain to download
// targetBlocks := blockCacheItems - 15
// hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
//
// // Set a sync init hook to catch progress changes
// starting := make(chan struct{})
// progress := make(chan struct{})
//
// tester.downloader.syncInitHook = func(origin, latest uint64) {
// starting <- struct{}{}
// <-progress
// }
// // Retrieve the sync progress and ensure they are zero (pristine sync)
// if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
// t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
// }
// // Synchronise half the blocks and check initial progress
// tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts)
// pending := new(sync.WaitGroup)
// pending.Add(1)
//
// go func() {
// defer pending.Done()
// if err := tester.sync("peer-half", nil, mode); err != nil {
// panic(fmt.Sprintf("failed to synchronise blocks: %v", err))
// }
// }()
// <-starting
// if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) {
// t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1)
// }
// progress <- struct{}{}
// pending.Wait()
//
// // Synchronise all the blocks and check continuation progress
// tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts)
// pending.Add(1)
//
// go func() {
// defer pending.Done()
// if err := tester.sync("peer-full", nil, mode); err != nil {
// panic(fmt.Sprintf("failed to synchronise blocks: %v", err))
// }
// }()
// <-starting
// if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) {
// t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
// }
// progress <- struct{}{}
// pending.Wait()
//
// // Check final progress after successful sync
// if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
// t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks)
// }
//}
//Tests that synchronisation progress (origin block number, current block number
//and highest block number) is tracked and updated correctly.
func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) }
func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) }
func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) }
func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) }
func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) }
func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) }
func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
t.Parallel()
tester := newTester()
defer tester.terminate()
// Create a small enough block chain to download
targetBlocks := blockCacheItems - 15
hashes, headers, blocks, receipts := tester.makeChain(targetBlocks, 0, tester.genesis, nil, false)
// Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
tester.downloader.syncInitHook = func(origin, latest uint64) {
starting <- struct{}{}
<-progress
}
// Retrieve the sync progress and ensure they are zero (pristine sync)
if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != 0 {
t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, 0)
}
// Synchronise half the blocks and check initial progress
tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("peer-half", nil, mode); err != nil {
panic(fmt.Sprintf("failed to synchronise blocks: %v", err))
}
}()
<-starting
if progress := tester.downloader.Progress(); progress.StartingBlock != 0 || progress.CurrentBlock != 0 || progress.HighestBlock != uint64(targetBlocks/2+1) {
t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, 0, 0, targetBlocks/2+1)
}
progress <- struct{}{}
pending.Wait()
// Synchronise all the blocks and check continuation progress
tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
if err := tester.sync("peer-full", nil, mode); err != nil {
panic(fmt.Sprintf("failed to synchronise blocks: %v", err))
}
}()
<-starting
if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks/2+1) || progress.HighestBlock != uint64(targetBlocks) {
t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
// Check final progress after successful sync
if progress := tester.downloader.Progress(); progress.StartingBlock != uint64(targetBlocks/2+1) || progress.CurrentBlock != uint64(targetBlocks) || progress.HighestBlock != uint64(targetBlocks) {
t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", progress.StartingBlock, progress.CurrentBlock, progress.HighestBlock, targetBlocks/2+1, targetBlocks, targetBlocks)
}
}
// Tests that synchronisation progress (origin block number and highest block
// number) is tracked and updated correctly in case of a fork (or manual head