consensus/bor: add span store abstraction for stateless verification (#1472)

Add a new abstraction called span store in bor consensus. This makes the header verification stateless as instead of fetching validator set (of a span) from genesis contract, we rely on span from heimdall itself for validator set. This doesn't change any assumptions for consensus as the data in genesis contracts is written by fetching span details from heimdall in the first place. 

This helps us with multiple things such as header verification for snap sync, validating producer set against header's extra data, and using heimdall as a source of truth instead of making a contract call to genesis contract which generally leads to issues when importing side chains (as we need to use the side chain state and not canonical). 

Span store is a minimal cache which maintains last 10 spans in memory. It provide simple read methods via span id and block number so that we can use it in consensus.

.golangci.yml

@@ -3,9 +3,6 @@
 run:
   timeout: 20m
   tests: true
-  # default is true. Enables skipping of directories:
-  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs-use-default: true
 
 linters:
   disable-all: true
@@ -21,7 +18,7 @@ linters:
     - staticcheck
     - bidichk
     - durationcheck
-    - exportloopref
+    - copyloopvar # replacement to exportloopref after go 1.22+
     - whitespace
     - revive # only certain checks enabled
 
@@ -50,6 +47,9 @@ linters-settings:
         exclude: [""]
 
 issues:
+  # default is true. Enables skipping of directories:
+  # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  exclude-dirs-use-default: true
   exclude-files:
     - core/genesis_alloc.go
   exclude-rules:

consensus/bor/bor.go

@@ -44,6 +44,8 @@ import (
 )
 
 const (
+	defaultSpanLength  = 6400 // Default span length i.e. number of bor blocks in a span
+	zerothSpanEnd      = 255  // End block of 0th span
 	checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database
 	inmemorySnapshots  = 128  // Number of recent vote snapshots to keep in memory
 	inmemorySignatures = 4096 // Number of recent block signatures to keep in memory
@@ -224,6 +226,8 @@ type Bor struct {
 	GenesisContractsClient GenesisContract
 	HeimdallClient         IHeimdallClient
 
+	spanStore SpanStore // Store to save previous span data from heimdall
+
 	// The fields below are for testing only
 	fakeDiff      bool // Skip difficulty verifications
 	devFakeAuthor bool
@@ -257,6 +261,9 @@ func New(
 	recents, _ := lru.NewARC(inmemorySnapshots)
 	signatures, _ := lru.NewARC(inmemorySignatures)
 
+	// Create a new span store
+	spanStore := NewSpanStore(heimdallClient, spanner, chainConfig.ChainID.String())
+
 	c := &Bor{
 		chainConfig:            chainConfig,
 		config:                 borConfig,
@@ -267,6 +274,7 @@ func New(
 		spanner:                spanner,
 		GenesisContractsClient: genesisContracts,
 		HeimdallClient:         heimdallClient,
+		spanStore:              spanStore,
 		devFakeAuthor:          devFakeAuthor,
 	}
 
@@ -467,14 +475,22 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainHeaderReader, header *t
 		return err
 	}
 
-	// Verify the validator list match the local contract
+	// Verify if the producer set in header's extra data matches with the list in span.
-	if IsSprintStart(number+1, c.config.CalculateSprint(number)) {
+	// We skip the check for 0th span as the producer set in contract v/s producer set
-		newValidators, err := c.spanner.GetCurrentValidatorsByBlockNrOrHash(context.Background(), rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), number+1)
+	// in heimdall span is different which will lead a mismatch. Moreover, to make the
-
+	// validation stateless, we use the span from heimdall (via span store) instead of
+	// span from validator set genesis contract as both are supposed to be equivalent.
+	if number > zerothSpanEnd && IsSprintStart(number+1, c.config.CalculateSprint(number)) {
+		span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
 		if err != nil {
 			return err
 		}
 
+		// Use producer set from span as it's equivalent to the data we get from genesis contract
+		newValidators := make([]*valset.Validator, len(span.SelectedProducers))
+		for i, val := range span.SelectedProducers {
+			newValidators[i] = &val
+		}
 		sort.Sort(valset.ValidatorsByAddress(newValidators))
 
 		headerVals, err := valset.ParseValidators(header.GetValidatorBytes(c.chainConfig))
@@ -561,8 +577,6 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co
 		// at a checkpoint block without a parent (light client CHT), or we have piled
 		// up more headers than allowed to be reorged (chain reinit from a freezer),
 		// consider the checkpoint trusted and snapshot it.
-
-		// TODO fix this
 		// nolint:nestif
 		if number == 0 {
 			checkpoint := chain.GetHeaderByNumber(number)
@@ -570,14 +584,14 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co
 				// get checkpoint data
 				hash := checkpoint.Hash()
 
-				// get validators and current span
+				// get validators from span
-				validators, err := c.spanner.GetCurrentValidatorsByHash(context.Background(), hash, number+1)
+				span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
 				if err != nil {
 					return nil, err
 				}
 
 				// new snap shot
-				snap = newSnapshot(c.chainConfig, c.signatures, number, hash, validators)
+				snap = newSnapshot(c.chainConfig, c.signatures, number, hash, span.ValidatorSet.Validators)
 				if err := snap.store(c.db); err != nil {
 					return nil, err
 				}
@@ -1161,7 +1175,7 @@ func (c *Bor) FetchAndCommitSpan(
 
 		heimdallSpan = *s
 	} else {
-		response, err := c.HeimdallClient.Span(ctx, newSpanID)
+		response, err := c.spanStore.spanById(ctx, newSpanID)
 		if err != nil {
 			return err
 		}
@@ -1292,6 +1306,8 @@ func validateEventRecord(eventRecord *clerk.EventRecordWithTime, number uint64,
 
 func (c *Bor) SetHeimdallClient(h IHeimdallClient) {
 	c.HeimdallClient = h
+	// Update the heimdall client in span store
+	c.spanStore.setHeimdallClient(h)
 }
 
 func (c *Bor) GetCurrentValidators(ctx context.Context, headerHash common.Hash, blockNumber uint64) ([]*valset.Validator, error) {

consensus/bor/snapshot.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
-	"github.com/ethereum/go-ethereum/log"
 
 	lru "github.com/hashicorp/golang-lru"
 
@@ -161,9 +160,12 @@ func (s *Snapshot) apply(headers []*types.Header, c *Bor) (*Snapshot, error) {
 			v.IncrementProposerPriority(1)
 
 			if v.CheckEmptyId() {
-				log.Warn("Empty id found on validator set. Querying on the validatorSet contract")
+				// Fetch the validator set from span
-				valsWithId, _ := c.spanner.GetCurrentValidatorsByHash(context.Background(), header.Hash(), number+1)
+				span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1)
-				v.IncludeIds(valsWithId)
+				if err != nil {
+					return nil, err
+				}
+				v.IncludeIds(span.ValidatorSet.Validators)
 			}
 			snap.ValidatorSet = v
 		}

consensus/bor/span_store.go

@@ -0,0 +1,174 @@
+package bor
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
+	"github.com/ethereum/go-ethereum/consensus/bor/valset"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+// maxSpanFetchLimit denotes maximum number of future spans to fetch. During snap sync,
+// we verify very large batch of headers. The maximum range is not known as of now and
+// hence we set a very high limit. It can be reduced later.
+const maxSpanFetchLimit = 10_000
+
+// SpanStore acts as a simple middleware to cache span data populated from heimdall. It is used
+// in multiple places of bor consensus for verification.
+type SpanStore struct {
+	store *lru.ARCCache
+
+	heimdallClient IHeimdallClient
+	spanner        Spanner
+
+	latestKnownSpanId uint64
+	chainId           string
+}
+
+func NewSpanStore(heimdallClient IHeimdallClient, spanner Spanner, chainId string) SpanStore {
+	cache, _ := lru.NewARC(10)
+	return SpanStore{
+		store:             cache,
+		heimdallClient:    heimdallClient,
+		spanner:           spanner,
+		latestKnownSpanId: 0,
+		chainId:           chainId,
+	}
+}
+
+// spanById returns a span given its id. It fetches span from heimdall if not found in cache.
+func (s *SpanStore) spanById(ctx context.Context, spanId uint64) (*span.HeimdallSpan, error) {
+	var currentSpan *span.HeimdallSpan
+	if value, ok := s.store.Get(spanId); ok {
+		currentSpan, _ = value.(*span.HeimdallSpan)
+	}
+
+	if currentSpan == nil {
+		var err error
+		if s.heimdallClient == nil {
+			if spanId == 0 {
+				currentSpan, err = getMockSpan0(ctx, s.spanner, s.chainId)
+				if err != nil {
+					log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err)
+					return nil, err
+				}
+			} else {
+				return nil, fmt.Errorf("unable to create test span without heimdall client for id %d", spanId)
+			}
+		} else {
+			currentSpan, err = s.heimdallClient.Span(ctx, spanId)
+			if err != nil {
+				log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err)
+				return nil, err
+			}
+		}
+		s.store.Add(spanId, currentSpan)
+		if currentSpan.Span.ID > s.latestKnownSpanId {
+			s.latestKnownSpanId = currentSpan.ID
+		}
+	}
+
+	return currentSpan, nil
+}
+
+// spanByBlockNumber returns a span given a block number. It fetches span from heimdall if not found in cache. It
+// assumes that a span has been committed before (i.e. is current or past span) and returns an error if
+// asked for a future span. This is safe to assume as we don't have a way to find out span id for a future block
+// unless we hardcode the span length (which we don't want to).
+func (s *SpanStore) spanByBlockNumber(ctx context.Context, blockNumber uint64) (*span.HeimdallSpan, error) {
+	// As we don't persist latest known span to db, we loose the value on restarts. This leads to multiple heimdall calls
+	// which can be avoided. Hence we estimate the span id from block number which updates the latest known span id. Note
+	// that we still check if the block number lies in the range of span before returning it.
+	estimatedSpanId := estimateSpanId(blockNumber)
+	// Ignore the return value of this span as we validate it later in the loop
+	_, err := s.spanById(ctx, estimatedSpanId)
+	if err != nil {
+		return nil, err
+	}
+	// Iterate over all spans and check for number. This is to replicate the behaviour implemented in
+	// https://github.com/maticnetwork/genesis-contracts/blob/master/contracts/BorValidatorSet.template#L118-L134
+	// This logic is independent of the span length (bit extra effort but maintains equivalence) and will work
+	// for all span lengths (even if we change it in future).
+	latestKnownSpanId := s.latestKnownSpanId
+	for id := int(latestKnownSpanId); id >= 0; id-- {
+		span, err := s.spanById(ctx, uint64(id))
+		if err != nil {
+			return nil, err
+		}
+		if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock {
+			return span, nil
+		}
+		// Check if block number given is out of bounds
+		if id == int(latestKnownSpanId) && blockNumber > span.EndBlock {
+			return getFutureSpan(ctx, uint64(id)+1, blockNumber, latestKnownSpanId, s)
+		}
+	}
+
+	return nil, fmt.Errorf("span not found for block %d", blockNumber)
+}
+
+// getFutureSpan fetches span for future block number. It is mostly needed during snap sync.
+func getFutureSpan(ctx context.Context, id uint64, blockNumber uint64, latestKnownSpanId uint64, s *SpanStore) (*span.HeimdallSpan, error) {
+	for {
+		if id > latestKnownSpanId+maxSpanFetchLimit {
+			return nil, fmt.Errorf("span not found for block %d", blockNumber)
+		}
+		span, err := s.spanById(ctx, id)
+		if err != nil {
+			return nil, err
+		}
+		if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock {
+			return span, nil
+		}
+		id++
+	}
+}
+
+// estimateSpanId returns the corresponding span id for the given block number in a deterministic way.
+func estimateSpanId(blockNumber uint64) uint64 {
+	if blockNumber > zerothSpanEnd {
+		return 1 + (blockNumber-zerothSpanEnd-1)/defaultSpanLength
+	}
+	return 0
+}
+
+// setHeimdallClient sets the underlying heimdall client to be used. It is useful in
+// tests where mock heimdall client is set after creation of bor instance explicitly.
+func (s *SpanStore) setHeimdallClient(client IHeimdallClient) {
+	s.heimdallClient = client
+}
+
+// getMockSpan0 constructs a mock span 0 by fetching validator set from genesis state. This should
+// only be used in tests where heimdall client is not available.
+func getMockSpan0(ctx context.Context, spanner Spanner, chainId string) (*span.HeimdallSpan, error) {
+	if spanner == nil {
+		return nil, fmt.Errorf("spanner not available to fetch validator set")
+	}
+
+	// Fetch validators from genesis state
+	vals, err := spanner.GetCurrentValidatorsByBlockNrOrHash(ctx, rpc.BlockNumberOrHashWithNumber(0), 0)
+	if err != nil {
+		return nil, err
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: vals,
+		Proposer:   vals[0],
+	}
+	selectedProducers := make([]valset.Validator, len(vals))
+	for _, v := range vals {
+		selectedProducers = append(selectedProducers, *v)
+	}
+	return &span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: selectedProducers,
+		ChainID:           chainId,
+	}, nil
+}

consensus/bor/span_store_test.go

@@ -0,0 +1,220 @@
+package bor
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
+	"github.com/stretchr/testify/require"
+)
+
+type MockHeimdallClient struct {
+}
+
+func (h *MockHeimdallClient) Span(ctx context.Context, spanID uint64) (*span.HeimdallSpan, error) {
+	// Throw error for span id 100
+	if spanID == 100 {
+		return nil, fmt.Errorf("unable to fetch span")
+	}
+
+	// For everything else, return hardcoded span assuming length 6400 (except for span 0)
+	if spanID == 0 {
+		return &span.HeimdallSpan{
+			Span: span.Span{
+				ID:         0,
+				StartBlock: 0,
+				EndBlock:   255,
+			},
+		}, nil
+	} else {
+		return &span.HeimdallSpan{
+			Span: span.Span{
+				ID:         spanID,
+				StartBlock: 6400*(spanID-1) + 256,
+				EndBlock:   6400*spanID + 255,
+			},
+		}, nil
+	}
+}
+
+func TestSpanStore_SpanById(t *testing.T) {
+	spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337")
+	ctx := context.Background()
+
+	type Testcase struct {
+		id         uint64
+		startBlock uint64
+		endBlock   uint64
+	}
+
+	testcases := []Testcase{
+		{id: 0, startBlock: 0, endBlock: 255},
+		{id: 1, startBlock: 256, endBlock: 6655},
+		{id: 2, startBlock: 6656, endBlock: 13055},
+	}
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanById(ctx, tc.id)
+			require.NoError(t, err, "err in spanById for id=%d", tc.id)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanById for id=%d", tc.id)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanById for id=%d", tc.id)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanById for id=%d", tc.id)
+		})
+	}
+
+	// Ensure cache is updated
+	keys := spanStore.store.Keys()
+	require.Len(t, keys, 3, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for a few more spans
+	for i := spanStore.latestKnownSpanId; i <= 20; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys = spanStore.store.Keys()
+	require.Len(t, keys, 10, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ensure we're still able to fetch old spans even though they're evicted from cache
+	span, err := spanStore.spanById(ctx, 0)
+	require.NoError(t, err, "err in spanById after eviction for id=0")
+	require.Equal(t, uint64(0), span.ID, "invalid id in spanById after eviction for id=0")
+	require.Equal(t, uint64(0), span.StartBlock, "invalid start block in spanById after eviction for id=0")
+	require.Equal(t, uint64(255), span.EndBlock, "invalid end block in spanById after eviction for id=0")
+
+	// Try fetching span 100 and ensure error is handled
+	span, err = spanStore.spanById(ctx, 100)
+	require.Error(t, err, "expected error in spanById for id=100")
+	require.Nil(t, span, "expected nil span in spanById for id=100")
+
+	// Ensure latest known span is still the old one
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+}
+
+func TestSpanStore_SpanByBlockNumber(t *testing.T) {
+	spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337")
+	ctx := context.Background()
+
+	type Testcase struct {
+		blockNumber uint64
+		id          uint64
+		startBlock  uint64
+		endBlock    uint64
+	}
+
+	// Insert a few spans
+	for i := spanStore.latestKnownSpanId; i < 3; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys := spanStore.store.Keys()
+	require.Len(t, keys, 3, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for current and past spans via block number
+	testcases := []Testcase{
+		{blockNumber: 0, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 1, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 255, id: 0, startBlock: 0, endBlock: 255},
+		{blockNumber: 256, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 257, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6000, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6655, id: 1, startBlock: 256, endBlock: 6655},
+		{blockNumber: 6656, id: 2, startBlock: 6656, endBlock: 13055},
+		{blockNumber: 10000, id: 2, startBlock: 6656, endBlock: 13055},
+		{blockNumber: 13055, id: 2, startBlock: 6656, endBlock: 13055},
+	}
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber)
+			require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber)
+		})
+	}
+
+	// Insert a few more spans to trigger eviction
+	for i := spanStore.latestKnownSpanId; i <= 20; i++ {
+		_, err := spanStore.spanById(ctx, i)
+		require.NoError(t, err, "err in spanById for id=%d", i)
+	}
+
+	// Ensure cache is updated
+	keys = spanStore.store.Keys()
+	require.Len(t, keys, 10, "invalid length of keys in span store")
+
+	// Ensure latest known span id is updated
+	require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store")
+
+	// Ask for current and past spans
+	testcases = append(testcases, Testcase{blockNumber: 57856, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 60000, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 64255, id: 10, startBlock: 57856, endBlock: 64255})
+	testcases = append(testcases, Testcase{blockNumber: 121856, id: 20, startBlock: 121856, endBlock: 128255})
+	testcases = append(testcases, Testcase{blockNumber: 122000, id: 20, startBlock: 121856, endBlock: 128255})
+	testcases = append(testcases, Testcase{blockNumber: 128255, id: 20, startBlock: 121856, endBlock: 128255})
+
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber)
+			require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber)
+			require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber)
+		})
+	}
+
+	// Asking for a future span
+	span, err := spanStore.spanByBlockNumber(ctx, 128256) // block 128256 belongs to span 21 (future span)
+	require.NoError(t, err, "err in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(21), span.ID, "invalid id in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(128256), span.StartBlock, "invalid start block in spanByBlockNumber for future block 128256")
+	require.Equal(t, uint64(134655), span.EndBlock, "invalid end block in spanByBlockNumber for future block 128256")
+}
+
+// Irrelevant to the tests above but necessary for interface compatibility
+func (h *MockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchCheckpointCount(ctx context.Context) (int64, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestone(ctx context.Context) (*milestone.Milestone, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchNoAckMilestone(ctx context.Context, milestoneID string) error {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchLastNoAckMilestone(ctx context.Context) (string, error) {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) FetchMilestoneID(ctx context.Context, milestoneID string) error {
+	panic("implement me")
+}
+func (h *MockHeimdallClient) Close() {
+	panic("implement me")
+}

miner/fake_miner.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/api"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -46,17 +47,14 @@ func NewBorDefaultMiner(t *testing.T) *DefaultBorMiner {
 	ethAPI := api.NewMockCaller(ctrl)
 	ethAPI.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
 
+	// Mock span 0 for heimdall
+	span0 := createMockSpanForTest(common.Address{0x1}, "1337")
+
 	spanner := bor.NewMockSpanner(ctrl)
-	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{
+	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes()
-		{
-			ID:               0,
-			Address:          common.Address{0x1},
-			VotingPower:      100,
-			ProposerPriority: 0,
-		},
-	}, nil).AnyTimes()
 
 	heimdallClient := mocks.NewMockIHeimdallClient(ctrl)
+	heimdallClient.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes()
 	heimdallClient.EXPECT().Close().Times(1)
 
 	genesisContracts := bor.NewMockGenesisContract(ctrl)
@@ -157,6 +155,32 @@ func NewFakeBor(t TensingObject, chainDB ethdb.Database, chainConfig *params.Cha
 	return bor.New(chainConfig, chainDB, ethAPIMock, spanner, heimdallClientMock, contractMock, false)
 }
 
+func createMockSpanForTest(address common.Address, chainId string) span.HeimdallSpan {
+	// Mock span 0 for heimdall calls
+	validator := valset.Validator{
+		ID:               0,
+		Address:          address,
+		VotingPower:      100,
+		ProposerPriority: 0,
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: []*valset.Validator{&validator},
+		Proposer:   &validator,
+	}
+	span0 := span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: []valset.Validator{validator},
+		ChainID:           chainId,
+	}
+
+	return span0
+}
+
 var (
 	// Test chain configurations
 	testTxPoolConfigBor legacypool.Config

miner/worker_test.go

@@ -383,17 +383,14 @@ func getFakeBorFromConfig(t *testing.T, chainConfig *params.ChainConfig) (consen
 	ethAPIMock := api.NewMockCaller(ctrl)
 	ethAPIMock.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes()
 
+	// Mock span 0 for heimdall
+	span0 := createMockSpanForTest(TestBankAddress, chainConfig.ChainID.String())
+
 	spanner := bor.NewMockSpanner(ctrl)
-	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{
+	spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes()
-		{
-			ID:               0,
-			Address:          TestBankAddress,
-			VotingPower:      100,
-			ProposerPriority: 0,
-		},
-	}, nil).AnyTimes()
 
 	heimdallClientMock := mocks.NewMockIHeimdallClient(ctrl)
+	heimdallClientMock.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes()
 	heimdallClientMock.EXPECT().Close().AnyTimes()
 
 	contractMock := bor.NewMockGenesisContract(ctrl)

tests/bor/bor_test.go

@@ -25,8 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
-	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
-	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -40,7 +38,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/tests/bor/mocks"
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
@@ -379,39 +376,23 @@ func TestInsertingSpanSizeBlocks(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
+	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
 	_, currentSpan := loadSpanFromFile(t)
 
-	h, ctrl := getMockedHeimdallClient(t, currentSpan)
+	// Create mock heimdall client
-	defer func() {
+	ctrl := gomock.NewController(t)
-		_bor.Close()
+	defer ctrl.Finish()
-		ctrl.Finish()
-	}()
-
-	h.EXPECT().Close().AnyTimes()
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{
-		Proposer:   currentSpan.SelectedProducers[0].Address,
-		StartBlock: big.NewInt(0),
-		EndBlock:   big.NewInt(int64(spanSize)),
-	}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{
-		Proposer:   currentSpan.SelectedProducers[0].Address,
-		StartBlock: big.NewInt(0),
-		EndBlock:   big.NewInt(int64(spanSize)),
-	}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
+	h := createMockHeimdall(ctrl, &span0, currentSpan)
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
 	_bor.SetHeimdallClient(h)
 
 	block := init.genesis.ToBlock()
-	// to := int64(block.Header().Time)
 
-	currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)}
+	currentValidators := span0.ValidatorSet.Validators
 
 	spanner := getMockedSpanner(t, currentValidators)
 	_bor.SetSpanner(spanner)
@@ -446,48 +427,29 @@ func TestFetchStateSyncEvents(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
-	// A. Insert blocks for 0th sprint
+	// Insert blocks for 0th sprint
 	block := init.genesis.ToBlock()
 
-	// B.1 Mock /bor/span/1
+	// Create a mock span 0
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+	currentValidators := span0.ValidatorSet.Validators
+
+	// Load mock span 0
 	res, _ := loadSpanFromFile(t)
 
-	currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)}
+	// reate mock bor spanner
-
 	spanner := getMockedSpanner(t, currentValidators)
 	_bor.SetSpanner(spanner)
 
-	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
+	// Create mock heimdall client
-	for i := uint64(1); i < sprintSize; i++ {
-		if IsSpanEnd(i) {
-			currentValidators = res.Result.ValidatorSet.Validators
-		}
-
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
-		insertNewBlock(t, chain, block)
-	}
-
-	// B. Before inserting 1st block of the next sprint, mock heimdall deps
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h := mocks.NewMockIHeimdallClient(ctrl)
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
-	h.EXPECT().Close().AnyTimes()
 
-	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes()
+	// Mock state sync events
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
-
-	// B.2 Mock State Sync events
 	fromID := uint64(1)
 	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
 	to := int64(chain.GetHeaderByNumber(0).Time)
@@ -500,6 +462,16 @@ func TestFetchStateSyncEvents(t *testing.T) {
 	h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes()
 	_bor.SetHeimdallClient(h)
 
+	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
+	for i := uint64(1); i < sprintSize; i++ {
+		if IsSpanEnd(i) {
+			currentValidators = res.Result.ValidatorSet.Validators
+		}
+
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		insertNewBlock(t, chain, block)
+	}
+
 	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
 
 	// Validate the state sync transactions set by consensus
@@ -525,30 +497,24 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
-	// Mock /bor/span/1
+	// Create a mock span 0
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
+	// Load mock span 1
 	res, _ := loadSpanFromFile(t)
 
+	spanner := getMockedSpanner(t, span0.ValidatorSet.Validators)
+	_bor.SetSpanner(spanner)
+
 	// add the block producer
 	res.Result.ValidatorSet.Validators = append(res.Result.ValidatorSet.Validators, valset.NewValidator(addr, 4500))
 
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h := mocks.NewMockIHeimdallClient(ctrl)
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
-	h.EXPECT().Close().AnyTimes()
-
-	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes()
-
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
-
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
 	// Mock State Sync events
 	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
@@ -573,18 +539,9 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	// Insert blocks for 0th sprint
 	block := init.genesis.ToBlock()
 
-	var currentValidators []*valset.Validator
+	// Set the current validators from span0
-
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i <= sprintSize; i++ {
-		if IsSpanEnd(i) {
-			currentValidators = res.Result.ValidatorSet.Validators
-		} else {
-			currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)}
-		}
-
-		spanner := getMockedSpanner(t, currentValidators)
-		_bor.SetSpanner(spanner)
-
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
 		insertNewBlock(t, chain, block)
 	}
@@ -605,15 +562,22 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes()
 
 	for i := sprintSize + 1; i <= spanSize; i++ {
+		// Update the validator set at the end of span and update the respective mocks
 		if IsSpanEnd(i) {
 			currentValidators = res.Result.ValidatorSet.Validators
+
+			// Set the spanner to point to new validator set
+			spanner := getMockedSpanner(t, currentValidators)
+			_bor.SetSpanner(spanner)
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
 		} else {
 			currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)}
 		}
 
-		spanner := getMockedSpanner(t, currentValidators)
-		_bor.SetSpanner(spanner)
-
 		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
 		insertNewBlock(t, chain, block)
 	}
@@ -631,32 +595,25 @@ func TestOutOfTurnSigning(t *testing.T) {
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
 	_, heimdallSpan := loadSpanFromFile(t)
 	proposer := valset.NewValidator(addr, 10)
 	heimdallSpan.ValidatorSet.Validators = append(heimdallSpan.ValidatorSet.Validators, proposer)
 
-	// add the block producer
+	ctrl := gomock.NewController(t)
-	h, ctrl := getMockedHeimdallClient(t, heimdallSpan)
 	defer ctrl.Finish()
 
-	h.EXPECT().Close().AnyTimes()
+	h := createMockHeimdall(ctrl, &span0, heimdallSpan)
-
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
-
+	_bor.SetHeimdallClient(h)
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
 
 	spanner := getMockedSpanner(t, heimdallSpan.ValidatorSet.Validators)
 	_bor.SetSpanner(spanner)
 
-	_bor.SetHeimdallClient(h)
-
 	block := init.genesis.ToBlock()
 
 	setDifficulty := func(header *types.Header) {
@@ -665,8 +622,19 @@ func TestOutOfTurnSigning(t *testing.T) {
 		}
 	}
 
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i < spanSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, setDifficulty)
+		// Update the validator set before sprint end (so that it is returned when called for next block)
+		// E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification
+		if i == sprintSize-1 {
+			currentValidators = heimdallSpan.ValidatorSet.Validators
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
+		}
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, setDifficulty)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -724,27 +692,23 @@ func TestSignerNotFound(t *testing.T) {
 	t.Parallel()
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
+
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
-
 	defer _bor.Close()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+
 	_, heimdallSpan := loadSpanFromFile(t)
 
-	h, ctrl := getMockedHeimdallClient(t, heimdallSpan)
+	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	h.EXPECT().Close().AnyTimes()
+	h := createMockHeimdall(ctrl, &span0, heimdallSpan)
-	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
-
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
-	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
-
-	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
-
-	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
-
 	_bor.SetHeimdallClient(h)
 
 	block := init.genesis.ToBlock()
@@ -777,6 +741,7 @@ func TestSignerNotFound(t *testing.T) {
 //     gasFeeCap - gasTipCap < baseFee.
 //  6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap).
 func TestEIP1559Transition(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -995,6 +960,7 @@ func TestEIP1559Transition(t *testing.T) {
 
 func TestBurnContract(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1210,6 +1176,7 @@ func TestBurnContract(t *testing.T) {
 
 func TestBurnContractContractFetch(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	config := params.BorUnittestChainConfig
 	config.Bor.BurntContract = map[string]string{
 		"10":  "0x000000000000000000000000000000000000aaab",
@@ -1281,6 +1248,7 @@ func TestBurnContractContractFetch(t *testing.T) {
 
 // EIP1559 is not supported without EIP155. An error is expected
 func TestEIP1559TransitionWithEIP155(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1354,6 +1322,7 @@ func TestEIP1559TransitionWithEIP155(t *testing.T) {
 // it is up to a user to use protected transactions. so if a transaction is unprotected no errors related to chainID are expected.
 // transactions are checked in 2 places: transaction pool and blockchain processor.
 func TestTransitionWithoutEIP155(t *testing.T) {
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	var (
 		aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa")
 
@@ -1440,22 +1409,41 @@ func TestTransitionWithoutEIP155(t *testing.T) {
 
 func TestJaipurFork(t *testing.T) {
 	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
+
 	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
-
 	_bor := engine.(*bor.Bor)
 	defer _bor.Close()
 
 	block := init.genesis.ToBlock()
 
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
 	res, _ := loadSpanFromFile(t)
 
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	h := createMockHeimdall(ctrl, &span0, &res.Result)
+	_bor.SetHeimdallClient(h)
+
 	spanner := getMockedSpanner(t, res.Result.ValidatorSet.Validators)
 	_bor.SetSpanner(spanner)
 
+	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i < sprintSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
+		// Update the validator set before sprint end (so that it is returned when called for next block)
+		// E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification
+		if i == sprintSize-1 {
+			currentValidators = res.Result.ValidatorSet.Validators
+
+			// Update the span0's validator set to new validator set. This will be used in verify header when we query
+			// span to compare validator's set with header's extradata. Even though our span store has old validator set
+			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
+			span0.ValidatorSet.Validators = currentValidators
+		}
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
 		insertNewBlock(t, chain, block)
 
 		if block.Number().Uint64() == init.genesis.Config.Bor.JaipurBlock.Uint64()-1 {

tests/bor/helper.go

@@ -23,6 +23,8 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall" //nolint:typecheck
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
@@ -358,6 +360,46 @@ func getMockedHeimdallClient(t *testing.T, heimdallSpan *span.HeimdallSpan) (*mo
 	return h, ctrl
 }
 
+func createMockSpan(address common.Address, chainId string) span.HeimdallSpan {
+	// Mock span 0 for heimdall calls
+	validator := valset.Validator{
+		ID:               0,
+		Address:          address,
+		VotingPower:      10,
+		ProposerPriority: 0,
+	}
+	validatorSet := valset.ValidatorSet{
+		Validators: []*valset.Validator{&validator},
+		Proposer:   &validator,
+	}
+	span0 := span.HeimdallSpan{
+		Span: span.Span{
+			ID:         0,
+			StartBlock: 0,
+			EndBlock:   255,
+		},
+		ValidatorSet:      validatorSet,
+		SelectedProducers: []valset.Validator{validator},
+		ChainID:           chainId,
+	}
+
+	return span0
+}
+
+func createMockHeimdall(ctrl *gomock.Controller, span0, span1 *span.HeimdallSpan) *mocks.MockIHeimdallClient {
+	h := mocks.NewMockIHeimdallClient(ctrl)
+
+	h.EXPECT().Close().AnyTimes()
+	h.EXPECT().Span(gomock.Any(), uint64(0)).Return(span0, nil).AnyTimes()
+	h.EXPECT().Span(gomock.Any(), uint64(1)).Return(span1, nil).AnyTimes()
+	h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes()
+	h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes()
+	h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes()
+	h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes()
+
+	return h
+}
+
 func getMockedSpanner(t *testing.T, validators []*valset.Validator) *bor.MockSpanner {
 	t.Helper()