go-ethereum/eth/downloader/skeleton_test.go

// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package downloader

import (
	"encoding/json"
	"errors"
	"fmt"
	"math/big"
	"sync/atomic"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/rawdb"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/eth/protocols/eth"
	"github.com/ethereum/go-ethereum/ethdb"
	"github.com/ethereum/go-ethereum/log"
)

// hookedBackfiller is a tester backfiller with all interface methods mocked and
// hooked so tests can implement only the things they need.
type hookedBackfiller struct {
	// suspendHook is an optional hook to be called when the filler is requested
	// to be suspended.
	suspendHook func() *types.Header

	// resumeHook is an optional hook to be called when the filler is requested
	// to be resumed.
	resumeHook func()
}

// newHookedBackfiller creates a hooked backfiller with all callbacks disabled,
// essentially acting as a noop.
func newHookedBackfiller() backfiller {
	return new(hookedBackfiller)
}

// suspend requests the backfiller to abort any running full or snap sync
// based on the skeleton chain as it might be invalid. The backfiller should
// gracefully handle multiple consecutive suspends without a resume, even
// on initial startup.
func (hf *hookedBackfiller) suspend() *types.Header {
	if hf.suspendHook != nil {
		return hf.suspendHook()
	}
	return nil // we don't really care about header cleanups for now
}

// resume requests the backfiller to start running fill or snap sync based on
// the skeleton chain as it has successfully been linked. Appending new heads
// to the end of the chain will not result in suspend/resume cycles.
func (hf *hookedBackfiller) resume() {
	if hf.resumeHook != nil {
		hf.resumeHook()
	}
}

// skeletonTestPeer is a mock peer that can only serve header requests from a
// pre-perated header chain (which may be arbitrarily wrong for testing).
//
// Requesting anything else from these peers will hard panic. Note, do *not*
// implement any other methods. We actually want to make sure that the skeleton
// syncer only depends on - and will only ever do so - on header requests.
type skeletonTestPeer struct {
	id      string          // Unique identifier of the mock peer
	headers []*types.Header // Headers to serve when requested

	serve func(origin uint64) []*types.Header // Hook to allow custom responses

	served  atomic.Uint64 // Number of headers served by this peer
	dropped atomic.Uint64 // Flag whether the peer was dropped (stop responding)
}

// newSkeletonTestPeer creates a new mock peer to test the skeleton sync with.
func newSkeletonTestPeer(id string, headers []*types.Header) *skeletonTestPeer {
	return &skeletonTestPeer{
		id:      id,
		headers: headers,
	}
}

// newSkeletonTestPeerWithHook creates a new mock peer to test the skeleton sync with,
// and sets an optional serve hook that can return headers for delivery instead
// of the predefined chain. Useful for emulating malicious behavior that would
// otherwise require dedicated peer types.
func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func(origin uint64) []*types.Header) *skeletonTestPeer {
	return &skeletonTestPeer{
		id:      id,
		headers: headers,
		serve:   serve,
	}
}

// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered
// origin; associated with a particular peer in the download tester. The returned
// function can be used to retrieve batches of headers from the particular peer.
func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) {
	// Since skeleton test peer are in-memory mocks, dropping the does not make
	// them inaccessible. As such, check a local `dropped` field to see if the
	// peer has been dropped and should not respond any more.
	if p.dropped.Load() != 0 {
		return nil, errors.New("peer already dropped")
	}
	// Skeleton sync retrieves batches of headers going backward without gaps.
	// This ensures we can follow a clean parent progression without any reorg
	// hiccups. There is no need for any other type of header retrieval, so do
	// panic if there's such a request.
	if !reverse || skip != 0 {
		// Note, if other clients want to do these kinds of requests, it's their
		// problem, it will still work. We just don't want *us* making complicated
		// requests without a very strong reason to.
		panic(fmt.Sprintf("invalid header retrieval: reverse %v, want true; skip %d, want 0", reverse, skip))
	}
	// If the skeleton syncer requests the genesis block, panic. Whilst it could
	// be considered a valid request, our code specifically should not request it
	// ever since we want to link up headers to an existing local chain, which at
	// worse will be the genesis.
	if int64(origin)-int64(amount) < 0 {
		panic(fmt.Sprintf("headers requested before (or at) genesis: origin %d, amount %d", origin, amount))
	}
	// To make concurrency easier, the skeleton syncer always requests fixed size
	// batches of headers. Panic if the peer is requested an amount other than the
	// configured batch size (apart from the request leading to the genesis).
	if amount > requestHeaders || (amount < requestHeaders && origin > uint64(amount)) {
		panic(fmt.Sprintf("non-chunk size header batch requested: requested %d, want %d, origin %d", amount, requestHeaders, origin))
	}
	// Simple reverse header retrieval. Fill from the peer's chain and return.
	// If the tester has a serve hook set, try to use that before falling back
	// to the default behavior.
	var headers []*types.Header
	if p.serve != nil {
		headers = p.serve(origin)
	}
	if headers == nil {
		headers = make([]*types.Header, 0, amount)
		if len(p.headers) > int(origin) { // Don't serve headers if we're missing the origin
			for i := 0; i < amount; i++ {
				// Consider nil headers as a form of attack and withhold them. Nil
				// cannot be decoded from RLP, so it's not possible to produce an
				// attack by sending/receiving those over eth.
				header := p.headers[int(origin)-i]
				if header == nil {
					continue
				}
				headers = append(headers, header)
			}
		}
	}
	p.served.Add(uint64(len(headers)))

	hashes := make([]common.Hash, len(headers))
	for i, header := range headers {
		hashes[i] = header.Hash()
	}
	// Deliver the headers to the downloader
	req := &eth.Request{
		Peer: p.id,
	}
	res := &eth.Response{
		Req:  req,
		Res:  (*eth.BlockHeadersRequest)(&headers),
		Meta: hashes,
		Time: 1,
		Done: make(chan error),
	}
	go func() {
		sink <- res
		if err := <-res.Done; err != nil {
			log.Warn("Skeleton test peer response rejected", "err", err)
			p.dropped.Add(1)
		}
	}()
	return req, nil
}

func (p *skeletonTestPeer) Head() (common.Hash, *big.Int) {
	panic("skeleton sync must not request the remote head")
}

func (p *skeletonTestPeer) RequestHeadersByHash(common.Hash, int, int, bool, chan *eth.Response) (*eth.Request, error) {
	panic("skeleton sync must not request headers by hash")
}

func (p *skeletonTestPeer) RequestBodies([]common.Hash, chan *eth.Response) (*eth.Request, error) {
	panic("skeleton sync must not request block bodies")
}

func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (*eth.Request, error) {
	panic("skeleton sync must not request receipts")
}

// Tests various sync initializations based on previous leftovers in the database
// and announced heads.
func TestSkeletonSyncInit(t *testing.T) {
	// Create a few key headers
	var (
		genesis  = &types.Header{Number: big.NewInt(0)}
		block49  = &types.Header{Number: big.NewInt(49)}
		block49B = &types.Header{Number: big.NewInt(49), Extra: []byte("B")}
		block50  = &types.Header{Number: big.NewInt(50), ParentHash: block49.Hash()}
	)
	tests := []struct {
		headers  []*types.Header // Database content (beside the genesis)
		oldstate []*subchain     // Old sync state with various interrupted subchains
		head     *types.Header   // New head header to announce to reorg to
		newstate []*subchain     // Expected sync state after the reorg
	}{
		// Completely empty database with only the genesis set. The sync is expected
		// to create a single subchain with the requested head.
		{
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 50}},
		},
		// Empty database with only the genesis set with a leftover empty sync
		// progress. This is a synthetic case, just for the sake of covering things.
		{
			oldstate: []*subchain{},
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 50}},
		},
		// A single leftover subchain is present, older than the new head. The
		// old subchain should be left as is and a new one appended to the sync
		// status.
		{
			oldstate: []*subchain{{Head: 10, Tail: 5}},
			head:     block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 10, Tail: 5},
			},
		},
		// Multiple leftover subchains are present, older than the new head. The
		// old subchains should be left as is and a new one appended to the sync
		// status.
		{
			oldstate: []*subchain{
				{Head: 20, Tail: 15},
				{Head: 10, Tail: 5},
			},
			head: block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 20, Tail: 15},
				{Head: 10, Tail: 5},
			},
		},
		// A single leftover subchain is present, newer than the new head. The
		// newer subchain should be deleted and a fresh one created for the head.
		{
			oldstate: []*subchain{{Head: 65, Tail: 60}},
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 50}},
		},
		// Multiple leftover subchain is present, newer than the new head. The
		// newer subchains should be deleted and a fresh one created for the head.
		{
			oldstate: []*subchain{
				{Head: 75, Tail: 70},
				{Head: 65, Tail: 60},
			},
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 50}},
		},

		// Two leftover subchains are present, one fully older and one fully
		// newer than the announced head. The head should delete the newer one,
		// keeping the older one.
		{
			oldstate: []*subchain{
				{Head: 65, Tail: 60},
				{Head: 10, Tail: 5},
			},
			head: block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 10, Tail: 5},
			},
		},
		// Multiple leftover subchains are present, some fully older and some
		// fully newer than the announced head. The head should delete the newer
		// ones, keeping the older ones.
		{
			oldstate: []*subchain{
				{Head: 75, Tail: 70},
				{Head: 65, Tail: 60},
				{Head: 20, Tail: 15},
				{Head: 10, Tail: 5},
			},
			head: block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 20, Tail: 15},
				{Head: 10, Tail: 5},
			},
		},
		// A single leftover subchain is present and the new head is extending
		// it with one more header. We expect the subchain head to be pushed
		// forward.
		{
			headers:  []*types.Header{block49},
			oldstate: []*subchain{{Head: 49, Tail: 5}},
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 5}},
		},
		// A single leftover subchain is present and although the new head does
		// extend it number wise, the hash chain does not link up. We expect a
		// new subchain to be created for the dangling head.
		{
			headers:  []*types.Header{block49B},
			oldstate: []*subchain{{Head: 49, Tail: 5}},
			head:     block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 49, Tail: 5},
			},
		},
		// A single leftover subchain is present. A new head is announced that
		// links into the middle of it, correctly anchoring into an existing
		// header. We expect the old subchain to be truncated and extended with
		// the new head.
		{
			headers:  []*types.Header{block49},
			oldstate: []*subchain{{Head: 100, Tail: 5}},
			head:     block50,
			newstate: []*subchain{{Head: 50, Tail: 5}},
		},
		// A single leftover subchain is present. A new head is announced that
		// links into the middle of it, but does not anchor into an existing
		// header. We expect the old subchain to be truncated and a new chain
		// be created for the dangling head.
		{
			headers:  []*types.Header{block49B},
			oldstate: []*subchain{{Head: 100, Tail: 5}},
			head:     block50,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
				{Head: 49, Tail: 5},
			},
		},
	}
	for i, tt := range tests {
		// Create a fresh database and initialize it with the starting state
		db := rawdb.NewMemoryDatabase()

		rawdb.WriteHeader(db, genesis)
		for _, header := range tt.headers {
			rawdb.WriteSkeletonHeader(db, header)
		}
		if tt.oldstate != nil {
			blob, _ := json.Marshal(&skeletonProgress{Subchains: tt.oldstate})
			rawdb.WriteSkeletonSyncStatus(db, blob)
		}
		// Create a skeleton sync and run a cycle
		wait := make(chan struct{})

		skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller())
		skeleton.syncStarting = func() { close(wait) }
		skeleton.Sync(tt.head, nil, true)

		<-wait
		skeleton.Terminate()

		// Ensure the correct resulting sync status
		expect := skeletonExpect{state: tt.newstate}
		if err := checkSkeletonProgress(db, false, nil, expect); err != nil {
			t.Errorf("test %d: %v", i, err)
		}
	}
}

// Tests that a running skeleton sync can be extended with properly linked up
// headers but not with side chains.
func TestSkeletonSyncExtend(t *testing.T) {
	// Create a few key headers
	var (
		genesis  = &types.Header{Number: big.NewInt(0)}
		block49  = &types.Header{Number: big.NewInt(49)}
		block49B = &types.Header{Number: big.NewInt(49), Extra: []byte("B")}
		block50  = &types.Header{Number: big.NewInt(50), ParentHash: block49.Hash()}
		block51  = &types.Header{Number: big.NewInt(51), ParentHash: block50.Hash()}
	)
	tests := []struct {
		head     *types.Header // New head header to announce to reorg to
		extend   *types.Header // New head header to announce to extend with
		newstate []*subchain   // Expected sync state after the reorg
		err      error         // Whether extension succeeds or not
	}{
		// Initialize a sync and try to extend it with a subsequent block.
		{
			head:   block49,
			extend: block50,
			newstate: []*subchain{
				{Head: 50, Tail: 49},
			},
		},
		// Initialize a sync and try to extend it with the existing head block.
		{
			head:   block49,
			extend: block49,
			newstate: []*subchain{
				{Head: 49, Tail: 49},
			},
		},
		// Initialize a sync and try to extend it with a sibling block.
		{
			head:   block49,
			extend: block49B,
			newstate: []*subchain{
				{Head: 49, Tail: 49},
			},
			err: errChainReorged,
		},
		// Initialize a sync and try to extend it with a number-wise sequential
		// header, but a hash wise non-linking one.
		{
			head:   block49B,
			extend: block50,
			newstate: []*subchain{
				{Head: 49, Tail: 49},
			},
			err: errChainForked,
		},
		// Initialize a sync and try to extend it with a non-linking future block.
		{
			head:   block49,
			extend: block51,
			newstate: []*subchain{
				{Head: 49, Tail: 49},
			},
			err: errChainGapped,
		},
		// Initialize a sync and try to extend it with a past canonical block.
		{
			head:   block50,
			extend: block49,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
			},
			err: errChainReorged,
		},
		// Initialize a sync and try to extend it with a past sidechain block.
		{
			head:   block50,
			extend: block49B,
			newstate: []*subchain{
				{Head: 50, Tail: 50},
			},
			err: errChainReorged,
		},
	}
	for i, tt := range tests {
		// Create a fresh database and initialize it with the starting state
		db := rawdb.NewMemoryDatabase()
		rawdb.WriteHeader(db, genesis)

		// Create a skeleton sync and run a cycle
		wait := make(chan struct{})

		skeleton := newSkeleton(db, newPeerSet(), nil, newHookedBackfiller())
		skeleton.syncStarting = func() { close(wait) }
		skeleton.Sync(tt.head, nil, true)

		<-wait
		if err := skeleton.Sync(tt.extend, nil, false); !errors.Is(err, tt.err) {
			t.Errorf("test %d: extension failure mismatch: have %v, want %v", i, err, tt.err)
		}
		skeleton.Terminate()

		// Ensure the correct resulting sync status
		expect := skeletonExpect{state: tt.newstate}
		if err := checkSkeletonProgress(db, false, nil, expect); err != nil {
			t.Errorf("test %d: %v", i, err)
		}
	}
}

type skeletonExpect struct {
	state []*subchain // Expected sync state after the post-init event
	serve uint64      // Expected number of header retrievals after initial cycle
	drop  uint64      // Expected number of peers dropped after initial cycle
}

type skeletonTest struct {
	fill          bool // Whether to run a real backfiller in this test case
	unpredictable bool // Whether to ignore drops/serves due to uncertain packet assignments

	head  *types.Header       // New head header to announce to reorg to
	peers []*skeletonTestPeer // Initial peer set to start the sync with
	mid   skeletonExpect

	newHead *types.Header     // New header to anoint on top of the old one
	newPeer *skeletonTestPeer // New peer to join the skeleton syncer
	end     skeletonExpect
}

// Tests that the skeleton sync correctly retrieves headers from one or more
// peers without duplicates or other strange side effects.
func TestSkeletonSyncRetrievals(t *testing.T) {
	//log.SetDefault(log.NewLogger(log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false))))

	// Since skeleton headers don't need to be meaningful, beyond a parent hash
	// progression, create a long fake chain to test with.
	chain := []*types.Header{{Number: big.NewInt(0)}}
	for i := 1; i < 10000; i++ {
		chain = append(chain, &types.Header{
			ParentHash: chain[i-1].Hash(),
			Number:     big.NewInt(int64(i)),
		})
	}
	// Some tests require a forking side chain to trigger cornercases.
	var sidechain []*types.Header
	for i := 0; i < len(chain)/2; i++ { // Fork at block #5000
		sidechain = append(sidechain, chain[i])
	}
	for i := len(chain) / 2; i < len(chain); i++ {
		sidechain = append(sidechain, &types.Header{
			ParentHash: sidechain[i-1].Hash(),
			Number:     big.NewInt(int64(i)),
			Extra:      []byte("B"), // force a different hash
		})
	}
	tests := []skeletonTest{
		// Completely empty database with only the genesis set. The sync is expected
		// to create a single subchain with the requested head. No peers however, so
		// the sync should be stuck without any progression.
		//
		// When a new peer is added, it should detect the join and fill the headers
		// to the genesis block.
		{
			head: chain[len(chain)-1],
			mid: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: uint64(len(chain) - 1)}},
			},

			newPeer: newSkeletonTestPeer("test-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}},
				serve: uint64(len(chain) - 2), // len - head - genesis
			},
		},
		// Completely empty database with only the genesis set. The sync is expected
		// to create a single subchain with the requested head. With one valid peer,
		// the sync is expected to complete already in the initial round.
		//
		// Adding a second peer should not have any effect.
		{
			head:  chain[len(chain)-1],
			peers: []*skeletonTestPeer{newSkeletonTestPeer("test-peer-1", chain)},
			mid: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}},
				serve: uint64(len(chain) - 2), // len - head - genesis
			},

			newPeer: newSkeletonTestPeer("test-peer-2", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}},
				serve: uint64(len(chain) - 2), // len - head - genesis
			},
		},
		// Completely empty database with only the genesis set. The sync is expected
		// to create a single subchain with the requested head. With many valid peers,
		// the sync is expected to complete already in the initial round.
		//
		// Adding a new peer should not have any effect.
		{
			head: chain[len(chain)-1],
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("test-peer-1", chain),
				newSkeletonTestPeer("test-peer-2", chain),
				newSkeletonTestPeer("test-peer-3", chain),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}},
				serve: uint64(len(chain) - 2), // len - head - genesis
			},

			newPeer: newSkeletonTestPeer("test-peer-4", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain) - 1), Tail: 1}},
				serve: uint64(len(chain) - 2), // len - head - genesis
			},
		},
		// This test checks if a peer tries to withhold a header - *on* the sync
		// boundary - instead of sending the requested amount. The malicious short
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100],
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-skipper", append(append(append([]*types.Header{}, chain[:99]...), nil), chain[100:]...)),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 3, // len - head - genesis - missing
				drop:  1,                        // penalize shortened header deliveries
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 3) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test checks if a peer tries to withhold a header - *off* the sync
		// boundary - instead of sending the requested amount. The malicious short
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100],
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-skipper", append(append(append([]*types.Header{}, chain[:50]...), nil), chain[51:]...)),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 3, // len - head - genesis - missing
				drop:  1,                        // penalize shortened header deliveries
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 3) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test checks if a peer tries to duplicate a header - *on* the sync
		// boundary - instead of sending the correct sequence. The malicious duped
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-duper", append(append(append([]*types.Header{}, chain[:99]...), chain[98]), chain[100:]...)),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 2, // len - head - genesis
				drop:  1,                        // penalize invalid header sequences
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test checks if a peer tries to duplicate a header - *off* the sync
		// boundary - instead of sending the correct sequence. The malicious duped
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-duper", append(append(append([]*types.Header{}, chain[:50]...), chain[49]), chain[51:]...)),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 2, // len - head - genesis
				drop:  1,                        // penalize invalid header sequences
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test checks if a peer tries to inject a different header - *on*
		// the sync boundary - instead of sending the correct sequence. The bad
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-changer",
					append(
						append(
							append([]*types.Header{}, chain[:99]...),
							&types.Header{
								ParentHash: chain[98].Hash(),
								Number:     big.NewInt(int64(99)),
								GasLimit:   1,
							},
						), chain[100:]...,
					),
				),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 2, // len - head - genesis
				drop:  1,                        // different set of headers, drop // TODO(karalabe): maybe just diff sync?
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test checks if a peer tries to inject a different header - *off*
		// the sync boundary - instead of sending the correct sequence. The bad
		// package should not be accepted.
		//
		// Joining with a new peer should however unblock the sync.
		{
			head: chain[requestHeaders+100], // We want to force the 100th header to be a request boundary
			peers: []*skeletonTestPeer{
				newSkeletonTestPeer("header-changer",
					append(
						append(
							append([]*types.Header{}, chain[:50]...),
							&types.Header{
								ParentHash: chain[49].Hash(),
								Number:     big.NewInt(int64(50)),
								GasLimit:   1,
							},
						), chain[51:]...,
					),
				),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 100}},
				serve: requestHeaders + 101 - 2, // len - head - genesis
				drop:  1,                        // different set of headers, drop
			},

			newPeer: newSkeletonTestPeer("good-peer", chain),
			end: skeletonExpect{
				state: []*subchain{{Head: requestHeaders + 100, Tail: 1}},
				serve: (requestHeaders + 101 - 2) + (100 - 1), // midserve + lenrest - genesis
				drop:  1,                                      // no new drops
			},
		},
		// This test reproduces a bug caught during review (kudos to @holiman)
		// where a subchain is merged with a previously interrupted one, causing
		// pending data in the scratch space to become "invalid" (since we jump
		// ahead during subchain merge). In that case it is expected to ignore
		// the queued up data instead of trying to process on top of a shifted
		// task set.
		//
		// The test is a bit convoluted since it needs to trigger a concurrency
		// issue. First we sync up an initial chain of 2x512 items. Then announce
		// 2x512+2 as head and delay delivering the head batch to fill the scratch
		// space first. The delivery head should merge with the previous download
		// and the scratch space must not be consumed further.
		{
			head: chain[2*requestHeaders],
			peers: []*skeletonTestPeer{
				newSkeletonTestPeerWithHook("peer-1", chain, func(origin uint64) []*types.Header {
					if origin == chain[2*requestHeaders+1].Number.Uint64() {
						time.Sleep(100 * time.Millisecond)
					}
					return nil // Fallback to default behavior, just delayed
				}),
				newSkeletonTestPeerWithHook("peer-2", chain, func(origin uint64) []*types.Header {
					if origin == chain[2*requestHeaders+1].Number.Uint64() {
						time.Sleep(100 * time.Millisecond)
					}
					return nil // Fallback to default behavior, just delayed
				}),
			},
			mid: skeletonExpect{
				state: []*subchain{{Head: 2 * requestHeaders, Tail: 1}},
				serve: 2*requestHeaders - 1, // len - head - genesis
			},

			newHead: chain[2*requestHeaders+2],
			end: skeletonExpect{
				state: []*subchain{{Head: 2*requestHeaders + 2, Tail: 1}},
				serve: 4 * requestHeaders,
			},
		},
		// This test reproduces a bug caught by (@rjl493456442) where a skeleton
		// header goes missing, causing the sync to get stuck and/or panic.
		//
		// The setup requires a previously successfully synced chain up to a block
		// height N. That results is a single skeleton header (block N) and a single
		// subchain (head N, Tail N) being stored on disk.
		//
		// The following step requires a new sync cycle to a new side chain of a
		// height higher than N, and an ancestor lower than N (e.g. N-2, N+2).
		// In this scenario, when processing a batch of headers, a link point of
		// N-2 will be found, meaning that N-1 and N have been overwritten.
		//
		// The link event triggers an early exit, noticing that the previous sub-
		// chain is a leftover and deletes it (with it's skeleton header N). But
		// since skeleton header N has been overwritten to the new side chain, we
		// end up losing it and creating a gap.
		{
			fill:          true,
			unpredictable: true, // We have good and bad peer too, bad may be dropped, test too short for certainty

			head:  chain[len(chain)/2+1], // Sync up until the sidechain common ancestor + 2
			peers: []*skeletonTestPeer{newSkeletonTestPeer("test-peer-oldchain", chain)},
			mid: skeletonExpect{
				state: []*subchain{{Head: uint64(len(chain)/2 + 1), Tail: 1}},
			},

			newHead: sidechain[len(sidechain)/2+3], // Sync up until the sidechain common ancestor + 4
			newPeer: newSkeletonTestPeer("test-peer-newchain", sidechain),
			end: skeletonExpect{
				state: []*subchain{{Head: uint64(len(sidechain)/2 + 3), Tail: uint64(len(chain) / 2)}},
			},
		},
	}
	for i, tt := range tests {
		// Create a fresh database and initialize it with the starting state
		db := rawdb.NewMemoryDatabase()

		rawdb.WriteBlock(db, types.NewBlockWithHeader(chain[0]))
		rawdb.WriteReceipts(db, chain[0].Hash(), chain[0].Number.Uint64(), types.Receipts{})

		// Create a peer set to feed headers through
		peerset := newPeerSet()
		for _, peer := range tt.peers {
			peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id)))
		}
		// Create a peer dropper to track malicious peers
		dropped := make(map[string]int)
		drop := func(peer string) {
			if p := peerset.Peer(peer); p != nil {
				p.peer.(*skeletonTestPeer).dropped.Add(1)
			}
			peerset.Unregister(peer)
			dropped[peer]++
		}
		// Create a backfiller if we need to run more advanced tests
		filler := newHookedBackfiller()
		if tt.fill {
			var filled *types.Header

			filler = &hookedBackfiller{
				resumeHook: func() {
					var progress skeletonProgress
					json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress)

					for progress.Subchains[0].Tail < progress.Subchains[0].Head {
						header := rawdb.ReadSkeletonHeader(db, progress.Subchains[0].Tail)

						rawdb.WriteBlock(db, types.NewBlockWithHeader(header))
						rawdb.WriteReceipts(db, header.Hash(), header.Number.Uint64(), types.Receipts{})

						rawdb.DeleteSkeletonHeader(db, header.Number.Uint64())

						progress.Subchains[0].Tail++
						progress.Subchains[0].Next = header.Hash()
					}
					filled = rawdb.ReadSkeletonHeader(db, progress.Subchains[0].Tail)

					rawdb.WriteBlock(db, types.NewBlockWithHeader(filled))
					rawdb.WriteReceipts(db, filled.Hash(), filled.Number.Uint64(), types.Receipts{})
				},

				suspendHook: func() *types.Header {
					prev := filled
					filled = nil

					return prev
				},
			}
		}
		// Create a skeleton sync and run a cycle
		skeleton := newSkeleton(db, peerset, drop, filler)
		skeleton.Sync(tt.head, nil, true)

		// Wait a bit (bleah) for the initial sync loop to go to idle. This might
		// be either a finish or a never-start hence why there's no event to hook.
		waitStart := time.Now()
		for waitTime := 20 * time.Millisecond; time.Since(waitStart) < 2*time.Second; waitTime = waitTime * 2 {
			time.Sleep(waitTime)
			if err := checkSkeletonProgress(db, tt.unpredictable, tt.peers, tt.mid); err == nil {
				break
			}
		}
		if err := checkSkeletonProgress(db, tt.unpredictable, tt.peers, tt.mid); err != nil {
			t.Errorf("test %d, mid: %v", i, err)
			continue
		}

		// Apply the post-init events if there's any
		endpeers := tt.peers
		if tt.newPeer != nil {
			if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil {
				t.Errorf("test %d: failed to register new peer: %v", i, err)
			}
			time.Sleep(time.Millisecond * 50) // given time for peer registration
			endpeers = append(tt.peers, tt.newPeer)
		}
		if tt.newHead != nil {
			skeleton.Sync(tt.newHead, nil, true)
		}

		// Wait a bit (bleah) for the second sync loop to go to idle. This might
		// be either a finish or a never-start hence why there's no event to hook.
		waitStart = time.Now()
		for waitTime := 20 * time.Millisecond; time.Since(waitStart) < 2*time.Second; waitTime = waitTime * 2 {
			time.Sleep(waitTime)
			if err := checkSkeletonProgress(db, tt.unpredictable, endpeers, tt.end); err == nil {
				break
			}
		}
		if err := checkSkeletonProgress(db, tt.unpredictable, endpeers, tt.end); err != nil {
			t.Errorf("test %d, end: %v", i, err)
			continue
		}
		// Check that the peers served no more headers than we actually needed
		// Clean up any leftover skeleton sync resources
		skeleton.Terminate()
	}
}

func checkSkeletonProgress(db ethdb.KeyValueReader, unpredictable bool, peers []*skeletonTestPeer, expected skeletonExpect) error {
	var progress skeletonProgress
	// Check the post-init end state if it matches the required results
	json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress)

	if len(progress.Subchains) != len(expected.state) {
		return fmt.Errorf("subchain count mismatch: have %d, want %d", len(progress.Subchains), len(expected.state))
	}
	for j := 0; j < len(progress.Subchains); j++ {
		if progress.Subchains[j].Head != expected.state[j].Head {
			return fmt.Errorf("subchain %d head mismatch: have %d, want %d", j, progress.Subchains[j].Head, expected.state[j].Head)
		}
		if progress.Subchains[j].Tail != expected.state[j].Tail {
			return fmt.Errorf("subchain %d tail mismatch: have %d, want %d", j, progress.Subchains[j].Tail, expected.state[j].Tail)
		}
	}
	if !unpredictable {
		var served uint64
		for _, peer := range peers {
			served += peer.served.Load()
		}
		if served != expected.serve {
			return fmt.Errorf("served headers mismatch: have %d, want %d", served, expected.serve)
		}
		var drops uint64
		for _, peer := range peers {
			drops += peer.dropped.Load()
		}
		if drops != expected.drop {
			return fmt.Errorf("dropped peers mismatch: have %d, want %d", drops, expected.drop)
		}
	}
	return nil
}