Merge branch 'develop' of https://github.com/maticnetwork/bor into upstream_merge_v1.15.7

.golangci.yml

@@ -18,7 +18,7 @@ linters:
     - staticcheck
     - bidichk
     - durationcheck
-    - copyloopvar
+    - copyloopvar # replacement to exportloopref after go 1.22+
     - whitespace
     - revive # only certain checks enabled
     - durationcheck
@@ -53,7 +53,7 @@ linters-settings:
 
 issues:
   # default is true. Enables skipping of directories:
-  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
   exclude-dirs-use-default: true
   exclude-files:
     - core/genesis_alloc.go

consensus/bor/bor.go

@@ -45,6 +45,8 @@ import (
 )
 
 const (
+	defaultSpanLength  = 6400 // Default span length i.e. number of bor blocks in a span
+	zerothSpanEnd      = 255  // End block of 0th span
 	checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database
 	inmemorySnapshots  = 128  // Number of recent vote snapshots to keep in memory
 	inmemorySignatures = 4096 // Number of recent block signatures to keep in memory
@@ -225,6 +227,8 @@ type Bor struct {
 	GenesisContractsClient GenesisContract
 	HeimdallClient         IHeimdallClient
 
+	spanStore SpanStore // Store to save previous span data from heimdall
+
 	// The fields below are for testing only
 	fakeDiff      bool // Skip difficulty verifications
 	devFakeAuthor bool
@@ -258,6 +262,9 @@ func New(
 	recents, _ := lru.NewARC(inmemorySnapshots)
 	signatures, _ := lru.NewARC(inmemorySignatures)
 
+	// Create a new span store
+	spanStore := NewSpanStore(heimdallClient, spanner, chainConfig.ChainID.String())
+
 	c := &Bor{
 		chainConfig:            chainConfig,
 		config:                 borConfig,
@@ -268,6 +275,7 @@ func New(
 		spanner:                spanner,
 		GenesisContractsClient: genesisContracts,
 		HeimdallClient:         heimdallClient,
+		spanStore:              spanStore,
 		devFakeAuthor:          devFakeAuthor,
 	}
 
@@ -468,14 +476,22 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainHeaderReader, header *t
 		return err
 	}
 
-	// Verify the validator list match the local contract
+	// Verify if the producer set in header's extra data matches with the list in span.
-	if IsSprintStart(number+1, c.config.CalculateSprint(number)) {
+	// We skip the check for 0th span as the producer set in contract v/s producer set
-		newValidators, err := c.spanner.GetCurrentValidatorsByBlockNrOrHash(context.Background(), rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), number+1)
+	// in heimdall span is different which will lead a mismatch. Moreover, to make the
-
+	// validation stateless, we use the span from heimdall (via span store) instead of
+	// span from validator set genesis contract as both are supposed to be equivalent.
+	if number > zerothSpanEnd && IsSprintStart(number+1, c.config.CalculateSprint(number)) {
+		span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
 		if err != nil {
 			return err
 		}
 
+		// Use producer set from span as it's equivalent to the data we get from genesis contract
+		newValidators := make([]*valset.Validator, len(span.SelectedProducers))
+		for i, val := range span.SelectedProducers {
+			newValidators[i] = &val
+		}
 		sort.Sort(valset.ValidatorsByAddress(newValidators))
 
 		headerVals, err := valset.ParseValidators(header.GetValidatorBytes(c.chainConfig))
@@ -562,8 +578,6 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co
 		// at a checkpoint block without a parent (light client CHT), or we have piled
 		// up more headers than allowed to be reorged (chain reinit from a freezer),
 		// consider the checkpoint trusted and snapshot it.
-
-		// TODO fix this
 		// nolint:nestif
 		if number == 0 {
 			checkpoint := chain.GetHeaderByNumber(number)
@@ -571,14 +585,14 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co
 				// get checkpoint data
 				hash := checkpoint.Hash()
 
-				// get validators and current span
+				// get validators from span
-				validators, err := c.spanner.GetCurrentValidatorsByHash(context.Background(), hash, number+1)
+				span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
 				if err != nil {
 					return nil, err
 				}
 
 				// new snap shot
-				snap = newSnapshot(c.chainConfig, c.signatures, number, hash, validators)
+				snap = newSnapshot(c.chainConfig, c.signatures, number, hash, span.ValidatorSet.Validators)
 				if err := snap.store(c.db); err != nil {
 					return nil, err
 				}
@@ -1166,7 +1180,7 @@ func (c *Bor) FetchAndCommitSpan(
 
 		heimdallSpan = *s
 	} else {
-		response, err := c.HeimdallClient.Span(ctx, newSpanID)
+		response, err := c.spanStore.spanById(ctx, newSpanID)
 		if err != nil {
 			return err
 		}
@@ -1297,6 +1311,8 @@ func validateEventRecord(eventRecord *clerk.EventRecordWithTime, number uint64,
 
 func (c *Bor) SetHeimdallClient(h IHeimdallClient) {
 	c.HeimdallClient = h
+	// Update the heimdall client in span store
+	c.spanStore.setHeimdallClient(h)
 }
 
 func (c *Bor) GetCurrentValidators(ctx context.Context, headerHash common.Hash, blockNumber uint64) ([]*valset.Validator, error) {

consensus/bor/snapshot.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
-	"github.com/ethereum/go-ethereum/log"
 
 	lru "github.com/hashicorp/golang-lru"
 
@@ -161,9 +160,12 @@ func (s *Snapshot) apply(headers []*types.Header, c *Bor) (*Snapshot, error) {
 			v.IncrementProposerPriority(1)
 
 			if v.CheckEmptyId() {
-				log.Warn("Empty id found on validator set. Querying on the validatorSet contract")
+				// Fetch the validator set from span
-				valsWithId, _ := c.spanner.GetCurrentValidatorsByHash(context.Background(), header.Hash(), number+1)
+				span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
-				v.IncludeIds(valsWithId)
+				if err != nil {
+					return nil, err
+				}
+				v.IncludeIds(span.ValidatorSet.Validators)
 			}
 			snap.ValidatorSet = v
 		}

consensus/bor/span_store.go

@@ -0,0 +1,174 @@
+package bor
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
+	"github.com/ethereum/go-ethereum/consensus/bor/valset"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+// maxSpanFetchLimit denotes maximum number of future spans to fetch. During snap sync,
+// we verify very large batch of headers. The maximum range is not known as of now and
+// hence we set a very high limit. It can be reduced later.
+const maxSpanFetchLimit = 10_000
+
+// SpanStore acts as a simple middleware to cache span data populated from heimdall. It is used
+// in multiple places of bor consensus for verification.
+type SpanStore struct {
+	store *lru.ARCCache
+
+	heimdallClient IHeimdallClient
+	spanner        Spanner
+
+	latestKnownSpanId uint64
+	chainId           string
+}
+
+func NewSpanStore(heimdallClient IHeimdallClient, spanner Spanner, chainId string) SpanStore {
+	cache, _ := lru.NewARC(10)
+	return SpanStore{
+		store:             cache,
+		heimdallClient:    heimdallClient,
+		spanner:           spanner,
+		latestKnownSpanId: 0,
+		chainId:           chainId,
+	}
+}
+
+// spanById returns a span given its id. It fetches span from heimdall if not found in cache.
+func (s *SpanStore) spanById(ctx context.Context, spanId uint64) (*span.HeimdallSpan, error) {
+	var currentSpan *span.HeimdallSpan
+	if value, ok := s.store.Get(spanId); ok {
+		currentSpan, _ = value.(*span.HeimdallSpan)
+	}
+
+	if currentSpan == nil {
+		var err error
+		if s.heimdallClient == nil {
+			if spanId == 0 {
+				currentSpan, err = getMockSpan0(ctx, s.spanner, s.chainId)
+				if err != nil {
+					log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err)
+					return nil, err
+				}
+			} else {
+				return nil, fmt.Errorf("unable to create test span without heimdall client for id %d", spanId)
+			}
+		} else {
+			currentSpan, err = s.heimdallClient.Span(ctx, spanId)
+			if err != nil {
+				log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err)
+				return nil, err
+			}
+		}
+		s.store.Add(spanId, currentSpan)
+		if currentSpan.Span.ID > s.latestKnownSpanId {
+			s.latestKnownSpanId = currentSpan.ID
+		}
+	}
+
+	return currentSpan, nil
+}
+
+// spanByBlockNumber returns a span given a block number. It fetches span from heimdall if not found in cache. It
+// assumes that a span has been committed before (i.e. is current or past span) and returns an error if
+// asked for a future span. This is safe to assume as we don't have a way to find out span id for a future block
+// unless we hardcode the span length (which we don't want to).
+func (s *SpanStore) spanByBlockNumber(ctx context.Context, blockNumber uint64) (*span.HeimdallSpan, error) {
+	// As we don't persist latest known span to db, we loose the value on restarts. This leads to multiple heimdall calls
+	// which can be avoided. Hence we estimate the span id from block number which updates the latest known span id. Note
+	// that we still check if the block number lies in the range of span before returning it.
+	estimatedSpanId := estimateSpanId(blockNumber)
+	// Ignore the return value of this span as we validate it later in the loop
+	_, err := s.spanById(ctx, estimatedSpanId)
+	if err != nil {
+		return nil, err
+	}
+	// Iterate over all spans and check for number. This is to replicate the behaviour implemented in
+	// https://github.com/maticnetwork/genesis-contracts/blob/master/contracts/BorValidatorSet.template#L118-L134
+	// This logic is independent of the span length (bit extra effort but maintains equivalence) and will work
+	// for all span lengths (even if we change it in future).
+	latestKnownSpanId := s.latestKnownSpanId
+	for id := int(latestKnownSpanId); id >= 0; id-- {
+		span, err := s.spanById(ctx, uint64(id))
+		if err != nil {
+			return nil, err
+		}
+		if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock {
+			return span, nil
+		}
+		// Check if block number given is out of bounds
+		if id == int(latestKnownSpanId) && blockNumber > span.EndBlock {
+			return getFutureSpan(ctx, uint64(id)+1, blockNumber, latestKnownSpanId, s)
+		}
+	}
+
+	return nil, fmt.Errorf("span not found for block %d", blockNumber)
+}
+
+// getFutureSpan fetches span for future block number. It is mostly needed during snap sync.
+func getFutureSpan(ctx context.Context, id uint64, blockNumber uint64, latestKnownSpanId uint64, s *SpanStore) (*span.HeimdallSpan, error) {
+	for {
+		if id > latestKnownSpanId+maxSpanFetchLimit {
+			return nil, fmt.Errorf("span not found for block %d", blockNumber)
+		}
+		span, err := s.spanById(ctx, id)
+		if err != nil {
+			return nil, err
+		}
+		if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock {
+			return span, nil
+		}
+		id++
+	}
+}
+
+// estimateSpanId returns the corresponding span id for the given block number in a deterministic way.
+func estimateSpanId(blockNumber uint64) uint64 {
+	if blockNumber > zerothSpanEnd {
+		return 1 + (blockNumber-zerothSpanEnd-1)/defaultSpanLength
+	}
+	return 0
+}
+
+// setHeimdallClient sets the underlying heimdall client to be used. It is useful in
+// tests where mock heimdall client is set after creation of bor instance explicitly.
+func (s *SpanStore) setHeimdallClient(client IHeimdallClient) {
+	s.heimdallClient = client
+}
+
+// getMockSpan0 constructs a mock span 0 by fetching validator set from genesis state. This should
+// only be used in tests where heimdall client is not available.
+func getMockSpan0(ctx context.Context, spanner Spanner, chainId string) (*span.HeimdallSpan, error) {
+	if spanner == nil {
+		return nil, fmt.Errorf("spanner not available to fetch validator set")
+	}
+
+	// Fetch validators from genesis state
+	vals, err := spanner.GetCurrentValidatorsByBlockNrOrHash(ctx, rpc.BlockNumberOrHashWithNumber(0), 0)
+	if err != nil {
+		return nil, err
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: vals,
+		Proposer:   vals[0],
+	}
+	selectedProducers := make([]valset.Validator, len(vals))
+	for _, v := range vals {
+		selectedProducers = append(selectedProducers, *v)
+	}
+	return &span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: selectedProducers,
+		ChainID:           chainId,
+	}, nil
+}

consensus/bor/span_store_test.go

@@ -0,0 +1,220 @@
+package bor
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
+	"github.com/stretchr/testify/require"
+)
+
+type MockHeimdallClient struct {
+}
+
+func (h *MockHeimdallClient) Span(ctx context.Context, spanID uint64) (*span.HeimdallSpan, error) {
+	// Throw error for span id 100
+	if spanID == 100 {
+		return nil, fmt.Errorf("unable to fetch span")
+	}
+
+	// For everything else, return hardcoded span assuming length 6400 (except for span 0)
+	if spanID == 0 {
+		return &span.HeimdallSpan{
+			Span: span.Span{
+				ID:         0,
+				StartBlock: 0,
+				EndBlock:   255,
+			},
+		}, nil
+	} else {
+		return &span.HeimdallSpan{
+			Span: span.Span{
+				ID:         spanID,
+				StartBlock: 6400*(spanID-1) + 256,
+				EndBlock:   6400*spanID + 255,
+			},
+		}, nil
+	}
+}
+
+func TestSpanStore_SpanById(t *testing.T) {
+	spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337")
+	ctx := context.Background()
+
+	type Testcase struct {
+		id         uint64
+		startBlock uint64
+		endBlock   uint64
+	}
+
+	testcases := []Testcase{
+		{id: 0, startBlock: 0, endBlock: 255},
+		{id: 1, startBlock: 256, endBlock: 6655},
+		{id: 2, startBlock: 6656, endBlock: 13055},
+	}
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanById(ctx, tc.id)
+			require.NoError(t, err, "err in spanById for id=%d", tc.id)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanById for id=%d", tc.id)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanById for id=%d", tc.id)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanById for id=%d", tc.id)
+		})
+	}
+
+	// Ensure cache is updated
+	keys := spanStore.store.Keys()
+	require.Len(t, keys, 3, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for a few more spans
+	for i := spanStore.latestKnownSpanId; i <= 20; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys = spanStore.store.Keys()
+	require.Len(t, keys, 10, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ensure we're still able to fetch old spans even though they're evicted from cache
+	span, err := spanStore.spanById(ctx, 0)
+	require.NoError(t, err, "err in spanById after eviction for id=0")
+	require.Equal(t, uint64(0), span.ID, "invalid id in spanById after eviction for id=0")
+	require.Equal(t, uint64(0), span.StartBlock, "invalid start block in spanById after eviction for id=0")
+	require.Equal(t, uint64(255), span.EndBlock, "invalid end block in spanById after eviction for id=0")
+
+	// Try fetching span 100 and ensure error is handled
+	span, err = spanStore.spanById(ctx, 100)
+	require.Error(t, err, "expected error in spanById for id=100")
+	require.Nil(t, span, "expected nil span in spanById for id=100")
+
+	// Ensure latest known span is still the old one
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+}
+
+func TestSpanStore_SpanByBlockNumber(t *testing.T) {
+	spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337")
+	ctx := context.Background()
+
+	type Testcase struct {
+		blockNumber uint64
+		id          uint64
+		startBlock  uint64
+		endBlock    uint64
+	}
+
+	// Insert a few spans
+	for i := spanStore.latestKnownSpanId; i < 3; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys := spanStore.store.Keys()
+	require.Len(t, keys, 3, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for current and past spans via block number
+	testcases := []Testcase{
+		{blockNumber: 0, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 1, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 255, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 256, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 257, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6000, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6655, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6656, id: 2, startBlock: 6656, endBlock: 13055},
+		{blockNumber: 10000, id: 2, startBlock: 6656, endBlock: 13055},
+		{blockNumber: 13055, id: 2, startBlock: 6656, endBlock: 13055},
+	}
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber)
+			require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber)
+		})
+	}
+
+	// Insert a few more spans to trigger eviction
+	for i := spanStore.latestKnownSpanId; i <= 20; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys = spanStore.store.Keys()
+	require.Len(t, keys, 10, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for current and past spans
+	testcases = append(testcases, Testcase{blockNumber: 57856, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 60000, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 64255, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 121856, id: 20, startBlock: 121856, endBlock: 128255})
+	testcases = append(testcases, Testcase{blockNumber: 122000, id: 20, startBlock: 121856, endBlock: 128255})
+	testcases = append(testcases, Testcase{blockNumber: 128255, id: 20, startBlock: 121856, endBlock: 128255})
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber)
+			require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber)
+		})
+	}
+
+	// Asking for a future span
+	span, err := spanStore.spanByBlockNumber(ctx, 128256) // block 128256 belongs to span 21 (future span)
+	require.NoError(t, err, "err in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(21), span.ID, "invalid id in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(128256), span.StartBlock, "invalid start block in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(134655), span.EndBlock, "invalid end block in spanByBlockNumber for future block 128256")
+}
+
+// Irrelevant to the tests above but necessary for interface compatibility
+func (h *MockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchCheckpointCount(ctx context.Context) (int64, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestone(ctx context.Context) (*milestone.Milestone, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchNoAckMilestone(ctx context.Context, milestoneID string) error {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchLastNoAckMilestone(ctx context.Context) (string, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestoneID(ctx context.Context, milestoneID string) error {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) Close() {
+	panic("implement me")
+}

docs/cli/example_config.toml

@@ -13,7 +13,7 @@ ancient = ""                    # Data directory for ancient chain segments (def
 keystore = ""                   # Path of the directory where keystores are located
 "rpc.batchlimit" = 100          # Maximum number of messages in a batch (default=100, use 0 for no limits)
 "rpc.returndatalimit" = 100000  # Maximum size (in bytes) a result of an rpc request could have (default=100000, use 0 for no limits)
-syncmode = "full"               # Blockchain sync mode (only "full" sync supported)
+syncmode = "full"               # Blockchain sync mode ("full" or "snap")
 gcmode = "full"                 # Blockchain garbage collection mode ("full", "archive")
 snapshot = true                 # Enables the snapshot-database mode
 "bor.logs" = false              # Enables bor log retrieval

docs/cli/server.md

@@ -90,7 +90,7 @@ The ```bor server``` command runs the Bor client.
 
 - ```state.scheme```: Scheme to use for storing ethereum state ('hash' or 'path') (default: path)
 
-- ```syncmode```: Blockchain sync mode (only "full" sync supported) (default: full)
+- ```syncmode```: Blockchain sync mode ("full" or "snap") (default: full)
 
 - ```verbosity```: Logging verbosity for the server (5=trace|4=debug|3=info|2=warn|1=error|0=crit) (default: 3)
 

eth/handler.go

@@ -177,19 +177,14 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		// * the last snap sync is not finished while user specifies a full sync this
 		//   time. But we don't have any recent state for full sync.
 		// In these cases however it's safe to reenable snap sync.
-
+		fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock()
-		// TODO - uncomment when we (Polygon-PoS, bor) have snap sync/pbss
+		if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 {
-		// fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock()
+			h.snapSync.Store(true)
-
+			log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete")
-		// TODO - uncomment when we (Polygon-PoS, bor) have snap sync/pbss
+		} else if !h.chain.HasState(fullBlock.Root) {
-		// For more info - https://github.com/ethereum/go-ethereum/pull/28171
+			h.snapSync.Store(true)
-		// if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 {
+			log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing")
-		// 	h.snapSync.Store(true)
+		}
-		// 	log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete")
-		// } else if !h.chain.HasState(fullBlock.Root) {
-		// 	h.snapSync.Store(true)
-		// 	log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing")
-		// }
 	} else {
 		head := h.chain.CurrentBlock()
 		if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) {

eth/sync.go

@@ -190,43 +190,35 @@ func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp {
 }
 
 func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) {
-	// TODO - uncomment when we (Polygon-PoS, bor) have snap sync/pbss
+	// If we're in snap sync mode, return that directly
-	/*
+	if cs.handler.snapSync.Load() {
-		// If we're in snap sync mode, return that directly
+		block := cs.handler.chain.CurrentSnapBlock()
-		if cs.handler.snapSync.Load() {
+		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-			block := cs.handler.chain.CurrentSnapBlock()
+		return downloader.SnapSync, td
-			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+	}
-			return downloader.SnapSync, td
-		}
-	*/
 
 	// We are probably in full sync, but we might have rewound to before the
 	// snap sync pivot, check if we should re-enable snap sync.
 	head := cs.handler.chain.CurrentBlock()
-	// TODO - uncomment when we (Polygon-PoS, bor) have snap sync/pbss
+	if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
-	/*
+		if head.Number.Uint64() < *pivot {
-		if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
-			if head.Number.Uint64() < *pivot {
-				block := cs.handler.chain.CurrentSnapBlock()
-				td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-				return downloader.SnapSync, td
-			}
-		}
-	*/
-
-	// TODO - uncomment when we (Polygon-PoS, bor) have snap sync/pbss
-	// For more info - https://github.com/ethereum/go-ethereum/pull/28171
-	/*
-		// We are in a full sync, but the associated head state is missing. To complete
-		// the head state, forcefully rerun the snap sync. Note it doesn't mean the
-		// persistent state is corrupted, just mismatch with the head block.
-		if !cs.handler.chain.HasState(head.Root) {
 			block := cs.handler.chain.CurrentSnapBlock()
 			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-			log.Info("Reenabled snap sync as chain is stateless")
 			return downloader.SnapSync, td
 		}
-	*/
+	}
+
+	// For more info - https://github.com/ethereum/go-ethereum/pull/28171
+	// We are in a full sync, but the associated head state is missing. To complete
+	// the head state, forcefully rerun the snap sync. Note it doesn't mean the
+	// persistent state is corrupted, just mismatch with the head block.
+	if !cs.handler.chain.HasState(head.Root) {
+		block := cs.handler.chain.CurrentSnapBlock()
+		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+		log.Info("Reenabled snap sync as chain is stateless")
+		return downloader.SnapSync, td
+	}
+
 	// Nope, we're really full syncing
 	td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64())
 

internal/cli/server/config.go

@@ -1161,10 +1161,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (*
 	case "full":
 		n.SyncMode = downloader.FullSync
 	case "snap":
-		// n.SyncMode = downloader.SnapSync // TODO(snap): Uncomment when we have snap sync working
+		n.SyncMode = downloader.SnapSync
-		n.SyncMode = downloader.FullSync
-
-		log.Warn("Bor doesn't support Snap Sync yet, switching to Full Sync mode")
 	default:
 		return nil, fmt.Errorf("sync mode '%s' not found", c.SyncMode)
 	}

internal/cli/server/flags.go

@@ -88,7 +88,7 @@ func (c *Command) Flags(config *Config) *flagset.Flagset {
 	})
 	f.StringFlag(&flagset.StringFlag{
 		Name:    "syncmode",
-		Usage:   `Blockchain sync mode (only "full" sync supported)`,
+		Usage:   `Blockchain sync mode ("full" or "snap")`,
 		Value:   &c.cliConfig.SyncMode,
 		Default: c.cliConfig.SyncMode,
 	})

miner/fake_miner.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/api"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -46,17 +47,14 @@ func NewBorDefaultMiner(t *testing.T) *DefaultBorMiner {
 	ethAPI := api.NewMockCaller(ctrl)
 	ethAPI.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
 
+	// Mock span 0 for heimdall
+	span0 := createMockSpanForTest(common.Address{0x1}, "1337")
+
 	spanner := bor.NewMockSpanner(ctrl)
-	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{
+	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes()
-		{
-			ID:               0,
-			Address:          common.Address{0x1},
-			VotingPower:      100,
-			ProposerPriority: 0,
-		},
-	}, nil).AnyTimes()
 
 	heimdallClient := mocks.NewMockIHeimdallClient(ctrl)
+	heimdallClient.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes()
 	heimdallClient.EXPECT().Close().Times(1)
 
 	genesisContracts := bor.NewMockGenesisContract(ctrl)
@@ -157,6 +155,32 @@ func NewFakeBor(t TensingObject, chainDB ethdb.Database, chainConfig *params.Cha
 	return bor.New(chainConfig, chainDB, ethAPIMock, spanner, heimdallClientMock, contractMock, false)
 }
 
+func createMockSpanForTest(address common.Address, chainId string) span.HeimdallSpan {
+	// Mock span 0 for heimdall calls
+	validator := valset.Validator{
+		ID:               0,
+		Address:          address,
+		VotingPower:      100,
+		ProposerPriority: 0,
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: []*valset.Validator{&validator},
+		Proposer:   &validator,
+	}
+	span0 := span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: []valset.Validator{validator},
+		ChainID:           chainId,
+	}
+
+	return span0
+}
+
 var (
 	// Test chain configurations
 	testTxPoolConfigBor legacypool.Config

miner/worker_test.go

@@ -383,17 +383,14 @@ func getFakeBorFromConfig(t *testing.T, chainConfig *params.ChainConfig) (consen
 	ethAPIMock := api.NewMockCaller(ctrl)
 	ethAPIMock.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
 
+	// Mock span 0 for heimdall
+	span0 := createMockSpanForTest(TestBankAddress, chainConfig.ChainID.String())
+
 	spanner := bor.NewMockSpanner(ctrl)
-	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{
+	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes()
-		{
-			ID:               0,
-			Address:          TestBankAddress,
-			VotingPower:      100,
-			ProposerPriority: 0,
-		},
-	}, nil).AnyTimes()
 
 	heimdallClientMock := mocks.NewMockIHeimdallClient(ctrl)
+	heimdallClientMock.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes()
 	heimdallClientMock.EXPECT().Close().AnyTimes()
 
 	contractMock := bor.NewMockGenesisContract(ctrl)

tests/bor/bor_test.go

@@ -25,8 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
-	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
-	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -40,7 +38,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/tests/bor/mocks"
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
@@ -379,39 +376,23 @@ func TestInsertingSpanSizeBlocks(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
+	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
 	_, currentSpan := loadSpanFromFile(t)
 
-	h, ctrl := getMockedHeimdallClient(t, currentSpan)
+	// Create mock heimdall client
-	defer func() {
+	ctrl := gomock.NewController(t)
-		_bor.Close()
+	defer ctrl.Finish()
-		ctrl.Finish()
-	}()
-
-	h.EXPECT().Close().AnyTimes()
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{
-		Proposer:   currentSpan.SelectedProducers[0].Address,
-		StartBlock: big.NewInt(0),
-		EndBlock:   big.NewInt(int64(spanSize)),
-	}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{
-		Proposer:   currentSpan.SelectedProducers[0].Address,
-		StartBlock: big.NewInt(0),
-		EndBlock:   big.NewInt(int64(spanSize)),
-	}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
+	h := createMockHeimdall(ctrl, &span0, currentSpan)
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
 	_bor.SetHeimdallClient(h)
 
 	block := init.genesis.ToBlock()
-	// to := int64(block.Header().Time)
 
-	currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)}
+	currentValidators := span0.ValidatorSet.Validators
 
 	spanner := getMockedSpanner(t, currentValidators)
 	_bor.SetSpanner(spanner)
@@ -446,48 +427,29 @@ func TestFetchStateSyncEvents(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
-	// A. Insert blocks for 0th sprint
+	// Insert blocks for 0th sprint
 	block := init.genesis.ToBlock()
 
-	// B.1 Mock /bor/span/1
+	// Create a mock span 0
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+	currentValidators := span0.ValidatorSet.Validators
+
+	// Load mock span 0
 	res, _ := loadSpanFromFile(t)
 
-	currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)}
+	// reate mock bor spanner
-
 	spanner := getMockedSpanner(t, currentValidators)
 	_bor.SetSpanner(spanner)
 
-	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
+	// Create mock heimdall client
-	for i := uint64(1); i < sprintSize; i++ {
-		if IsSpanEnd(i) {
-			currentValidators = res.Result.ValidatorSet.Validators
-		}
-
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
-		insertNewBlock(t, chain, block)
-	}
-
-	// B. Before inserting 1st block of the next sprint, mock heimdall deps
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h := mocks.NewMockIHeimdallClient(ctrl)
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
-	h.EXPECT().Close().AnyTimes()
 
-	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes()
+	// Mock state sync events
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
-
-	// B.2 Mock State Sync events
 	fromID := uint64(1)
 	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
 	to := int64(chain.GetHeaderByNumber(0).Time)
@@ -500,6 +462,16 @@ func TestFetchStateSyncEvents(t *testing.T) {
 	h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes()
 	_bor.SetHeimdallClient(h)
 
+	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
+	for i := uint64(1); i < sprintSize; i++ {
+		if IsSpanEnd(i) {
+			currentValidators = res.Result.ValidatorSet.Validators
+		}
+
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		insertNewBlock(t, chain, block)
+	}
+
 	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
 
 	// Validate the state sync transactions set by consensus
@@ -525,30 +497,24 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
-	// Mock /bor/span/1
+	// Create a mock span 0
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
+	// Load mock span 1
 	res, _ := loadSpanFromFile(t)
 
+	spanner := getMockedSpanner(t, span0.ValidatorSet.Validators)
+	_bor.SetSpanner(spanner)
+
 	// add the block producer
 	res.Result.ValidatorSet.Validators = append(res.Result.ValidatorSet.Validators, valset.NewValidator(addr, 4500))
 
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h := mocks.NewMockIHeimdallClient(ctrl)
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
-	h.EXPECT().Close().AnyTimes()
-
-	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes()
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
 	// Mock State Sync events
 	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
@@ -573,18 +539,9 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	// Insert blocks for 0th sprint
 	block := init.genesis.ToBlock()
 
-	var currentValidators []*valset.Validator
+	// Set the current validators from span0
-
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i <= sprintSize; i++ {
-		if IsSpanEnd(i) {
-			currentValidators = res.Result.ValidatorSet.Validators
-		} else {
-			currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)}
-		}
-
-		spanner := getMockedSpanner(t, currentValidators)
-		_bor.SetSpanner(spanner)
-
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
 		insertNewBlock(t, chain, block)
 	}
@@ -605,15 +562,22 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes()
 
 	for i := sprintSize + 1; i <= spanSize; i++ {
+		// Update the validator set at the end of span and update the respective mocks
 		if IsSpanEnd(i) {
 			currentValidators = res.Result.ValidatorSet.Validators
+
+			// Set the spanner to point to new validator set
+			spanner := getMockedSpanner(t, currentValidators)
+			_bor.SetSpanner(spanner)
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
 		} else {
 			currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)}
 		}
 
-		spanner := getMockedSpanner(t, currentValidators)
-		_bor.SetSpanner(spanner)
-
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
 		insertNewBlock(t, chain, block)
 	}
@@ -631,32 +595,25 @@ func TestOutOfTurnSigning(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
 	_, heimdallSpan := loadSpanFromFile(t)
 	proposer := valset.NewValidator(addr, 10)
 	heimdallSpan.ValidatorSet.Validators = append(heimdallSpan.ValidatorSet.Validators, proposer)
 
-	// add the block producer
+	ctrl := gomock.NewController(t)
-	h, ctrl := getMockedHeimdallClient(t, heimdallSpan)
 	defer ctrl.Finish()
 
-	h.EXPECT().Close().AnyTimes()
+	h := createMockHeimdall(ctrl, &span0, heimdallSpan)
-
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
-
+	_bor.SetHeimdallClient(h)
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
 	spanner := getMockedSpanner(t, heimdallSpan.ValidatorSet.Validators)
 	_bor.SetSpanner(spanner)
 
-	_bor.SetHeimdallClient(h)
-
 	block := init.genesis.ToBlock()
 
 	setDifficulty := func(header *types.Header) {
@@ -665,8 +622,19 @@ func TestOutOfTurnSigning(t *testing.T) {
 		}
 	}
 
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i < spanSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, setDifficulty)
+		// Update the validator set before sprint end (so that it is returned when called for next block)
+		// E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification
+		if i == sprintSize-1 {
+			currentValidators = heimdallSpan.ValidatorSet.Validators
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
+		}
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, setDifficulty)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -724,27 +692,23 @@ func TestSignerNotFound(t *testing.T) {
 	t.Parallel()
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
+
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
 	_, heimdallSpan := loadSpanFromFile(t)
 
-	h, ctrl := getMockedHeimdallClient(t, heimdallSpan)
+	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h.EXPECT().Close().AnyTimes()
+	h := createMockHeimdall(ctrl, &span0, heimdallSpan)
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
-
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
-
 	_bor.SetHeimdallClient(h)
 
 	block := init.genesis.ToBlock()
@@ -777,6 +741,7 @@ func TestSignerNotFound(t *testing.T) {
 //     gasFeeCap - gasTipCap < baseFee.
 //  6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap).
 func TestEIP1559Transition(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -995,6 +960,7 @@ func TestEIP1559Transition(t *testing.T) {
 
 func TestBurnContract(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1210,6 +1176,7 @@ func TestBurnContract(t *testing.T) {
 
 func TestBurnContractContractFetch(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	config := params.BorUnittestChainConfig
 	config.Bor.BurntContract = map[string]string{
 		"10":  "0x000000000000000000000000000000000000aaab",
@@ -1281,6 +1248,7 @@ func TestBurnContractContractFetch(t *testing.T) {
 
 // EIP1559 is not supported without EIP155. An error is expected
 func TestEIP1559TransitionWithEIP155(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1354,6 +1322,7 @@ func TestEIP1559TransitionWithEIP155(t *testing.T) {
 // it is up to a user to use protected transactions. so if a transaction is unprotected no errors related to chainID are expected.
 // transactions are checked in 2 places: transaction pool and blockchain processor.
 func TestTransitionWithoutEIP155(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1440,22 +1409,41 @@ func TestTransitionWithoutEIP155(t *testing.T) {
 
 func TestJaipurFork(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
+
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
-
 	_bor := engine.(*bor.Bor)
 	defer _bor.Close()
 
 	block := init.genesis.ToBlock()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
 	res, _ := loadSpanFromFile(t)
 
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
+	_bor.SetHeimdallClient(h)
+
 	spanner := getMockedSpanner(t, res.Result.ValidatorSet.Validators)
 	_bor.SetSpanner(spanner)
 
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i < sprintSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
+		// Update the validator set before sprint end (so that it is returned when called for next block)
+		// E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification
+		if i == sprintSize-1 {
+			currentValidators = res.Result.ValidatorSet.Validators
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
+		}
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
 		insertNewBlock(t, chain, block)
 
 		if block.Number().Uint64() == init.genesis.Config.Bor.JaipurBlock.Uint64()-1 {

tests/bor/helper.go

@@ -23,6 +23,8 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall" //nolint:typecheck
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
@@ -358,6 +360,46 @@ func getMockedHeimdallClient(t *testing.T, heimdallSpan *span.HeimdallSpan) (*mo
 	return h, ctrl
 }
 
+func createMockSpan(address common.Address, chainId string) span.HeimdallSpan {
+	// Mock span 0 for heimdall calls
+	validator := valset.Validator{
+		ID:               0,
+		Address:          address,
+		VotingPower:      10,
+		ProposerPriority: 0,
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: []*valset.Validator{&validator},
+		Proposer:   &validator,
+	}
+	span0 := span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: []valset.Validator{validator},
+		ChainID:           chainId,
+	}
+
+	return span0
+}
+
+func createMockHeimdall(ctrl *gomock.Controller, span0, span1 *span.HeimdallSpan) *mocks.MockIHeimdallClient {
+	h := mocks.NewMockIHeimdallClient(ctrl)
+
+	h.EXPECT().Close().AnyTimes()
+	h.EXPECT().Span(gomock.Any(), uint64(0)).Return(span0, nil).AnyTimes()
+	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(span1, nil).AnyTimes()
+	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
+	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
+	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
+
+	return h
+}
+
 func getMockedSpanner(t *testing.T, validators []*valset.Validator) *bor.MockSpanner {
 	t.Helper()