consensus/bor: allow early block announcements (#1545)

This PR enables PIP-66. It allows primary block producers to announce blocks early before the header time for optimal time utilisation and preventing reorgs and p2p delays.

---------

Co-authored-by: Pratik Patil <pratikspatil024@gmail.com>

consensus/bor/bor.go

@@ -348,10 +348,24 @@ func (c *Bor) verifyHeader(chain consensus.ChainHeaderReader, header *types.Head
 	}
 
 	number := header.Number.Uint64()
+	now := uint64(time.Now().Unix())
 
-	// Don't waste time checking blocks from the future
+	// Allow early blocks if Bhilai HF is enabled
-	if header.Time > uint64(time.Now().Unix()) {
+	if c.config.IsBhilai(header.Number) {
-		return consensus.ErrFutureBlock
+		// Don't waste time checking blocks from the future but allow a buffer of block time for
+		// early block announcements. Note that this is a loose check and would allow early blocks
+		// from non-primary producer. Such blocks will be rejected later when we know the succession
+		// number of the signer in the current sprint.
+		if header.Time-c.config.CalculatePeriod(number) > now {
+			log.Error("Block announced too early post bhilai", "number", number, "headerTime", header.Time, "now", now)
+			return consensus.ErrFutureBlock
+		}
+	} else {
+		// Don't waste time checking blocks from the future
+		if header.Time > now {
+			log.Error("Block announced too early", "number", number, "headerTime", header.Time, "now", now)
+			return consensus.ErrFutureBlock
+		}
 	}
 
 	if err := validateHeaderExtraField(header.Extra); err != nil {
@@ -709,7 +723,16 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header
 		parent = chain.GetHeader(header.ParentHash, number-1)
 	}
 
-	if IsBlockOnTime(parent, header, number, succession, c.config) {
+	// Post Bhilai HF, reject blocks form non-primary producers if they're earlier than the expected time
+	if c.config.IsBhilai(header.Number) && succession != 0 {
+		now := uint64(time.Now().Unix())
+		if header.Time > now {
+			log.Error("Block announced too early by non-primary producer post bhilai", "number", number, "headerTime", header.Time, "now", now)
+			return consensus.ErrFutureBlock
+		}
+	}
+
+	if IsBlockEarly(parent, header, number, succession, c.config) {
 		return &BlockTooSoonError{number, succession}
 	}
 
@@ -724,7 +747,9 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header
 	return nil
 }
 
-func IsBlockOnTime(parent *types.Header, header *types.Header, number uint64, succession int, cfg *params.BorConfig) bool {
+// IsBlockEarly returns true if the header time is earlier than expected (according to consensus rules). This
+// can happen if the producer maliciously updates the header time.
+func IsBlockEarly(parent *types.Header, header *types.Header, number uint64, succession int, cfg *params.BorConfig) bool {
 	return parent != nil && header.Time < parent.Time+CalcProducerDelay(number, succession, cfg)
 }
 
@@ -827,6 +852,16 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header) e
 	header.Time = parent.Time + CalcProducerDelay(number, succession, c.config)
 	if header.Time < uint64(time.Now().Unix()) {
 		header.Time = uint64(time.Now().Unix())
+	} else {
+		// For primary validators, wait until the current block production window
+		// starts. This prevents bor from starting to build next block before time
+		// as we'd like to wait for new transactions. Although this change doesn't
+		// need a check for hard fork as it doesn't change any consensus rules, we
+		// still keep it for safety and testing.
+		if c.config.IsBhilai(big.NewInt(int64(number))) && succession == 0 {
+			startTime := time.Unix(int64(header.Time-c.config.CalculatePeriod(number)), 0)
+			time.Sleep(time.Until(startTime))
+		}
 	}
 
 	return nil
@@ -1021,8 +1056,20 @@ func (c *Bor) Seal(chain consensus.ChainHeaderReader, block *types.Block, result
 		return err
 	}
 
+	var delay time.Duration
+
 	// Sweet, the protocol permits us to sign the block, wait for our time
-	delay := time.Unix(int64(header.Time), 0).Sub(time.Now()) // nolint: gosimple
+	if c.config.IsBhilai(header.Number) {
+		delay = time.Until(time.Unix(int64(header.Time), 0)) // Wait until we reach header time for non-primary validators
+		if successionNumber == 0 {
+			// For primary producers, set the delay to `header.Time - block time` instead of `header.Time`
+			// for early block announcement instead of waiting for full block time.
+			delay = time.Until(time.Unix(int64(header.Time-c.config.CalculatePeriod(number)), 0))
+		}
+	} else {
+		delay = time.Until(time.Unix(int64(header.Time), 0)) // Wait until we reach header time
+	}
+
 	// wiggle was already accounted for in header.Time, this is just for logging
 	wiggle := time.Duration(successionNumber) * time.Duration(c.config.CalculateBackupMultiplier(number)) * time.Second
 

tests/bor/bor_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/fdlimit"
+	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/bor"
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
 	"github.com/ethereum/go-ethereum/consensus/bor/valset"
@@ -49,7 +50,6 @@ var (
 )
 
 func TestValidatorWentOffline(t *testing.T) {
-
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
 
@@ -212,7 +212,6 @@ func TestValidatorWentOffline(t *testing.T) {
 }
 
 func TestForkWithBlockTime(t *testing.T) {
-
 	cases := []struct {
 		name          string
 		sprint        map[string]uint64
@@ -372,7 +371,11 @@ func TestInsertingSpanSizeBlocks(t *testing.T) {
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
 
-	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize}
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
@@ -399,7 +402,7 @@ func TestInsertingSpanSizeBlocks(t *testing.T) {
 
 	// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
 	for i := uint64(1); i <= spanSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -423,7 +426,12 @@ func TestFetchStateSyncEvents(t *testing.T) {
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
 
-	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
+	stateSyncConfirmationDelay := int64(128)
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": uint64(stateSyncConfirmationDelay)}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize}
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
@@ -451,8 +459,12 @@ func TestFetchStateSyncEvents(t *testing.T) {
 
 	// Mock state sync events
 	fromID := uint64(1)
-	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
+	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time])
-	to := int64(chain.GetHeaderByNumber(0).Time)
+	// as indore hf is enabled, we need to consider the stateSyncConfirmationDelay and
+	// we need to predict the time of 4th block (i.e. the sprint end block) to calculate
+	// the correct value of to. As per the config, non sprint end primary blocks take
+	// 1s and sprint end ones take 6s. This leads to 3*1 + 6 = 9s of added time from genesis.
+	to := int64(chain.GetHeaderByNumber(0).Time) + 9 - stateSyncConfirmationDelay
 	eventCount := 50
 
 	sample := getSampleEventRecord(t)
@@ -468,11 +480,11 @@ func TestFetchStateSyncEvents(t *testing.T) {
 			currentValidators = res.Result.ValidatorSet.Validators
 		}
 
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false)
 		insertNewBlock(t, chain, block)
 	}
 
-	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators, false)
 
 	// Validate the state sync transactions set by consensus
 	validateStateSyncEvents(t, eventRecords, chain.GetStateSync())
@@ -493,7 +505,12 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
 
-	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
+	stateSyncConfirmationDelay := int64(128)
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": uint64(stateSyncConfirmationDelay)}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize}
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
@@ -517,9 +534,13 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	h := createMockHeimdall(ctrl, &span0, &res.Result)
 
 	// Mock State Sync events
-	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
+	// at # sprintSize, events are fetched for [fromID, (block-sprint).Time])
+	// as indore hf is enabled, we need to consider the stateSyncConfirmationDelay and
+	// we need to predict the time of 4th block (i.e. the sprint end block) to calculate
+	// the correct value of to. As per the config, non sprint end primary blocks take
+	// 1s and sprint end ones take 6s. This leads to 3*1 + 6 = 9s of added time from genesis.
 	fromID := uint64(1)
-	to := int64(chain.GetHeaderByNumber(0).Time)
+	to := int64(chain.GetHeaderByNumber(0).Time) + 9 - stateSyncConfirmationDelay
 	sample := getSampleEventRecord(t)
 
 	// First query will be from [id=1, (block-sprint).Time]
@@ -542,7 +563,7 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	// Set the current validators from span0
 	currentValidators := span0.ValidatorSet.Validators
 	for i := uint64(1); i <= sprintSize; i++ {
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -551,9 +572,9 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 	// state 6 was not written
 	require.Equal(t, uint64(4), lastStateID.Uint64())
 
-	//
+	// Same calculation for from and to as above
 	fromID = uint64(5)
-	to = int64(chain.GetHeaderByNumber(sprintSize).Time)
+	to = int64(chain.GetHeaderByNumber(sprintSize).Time) + 9 - stateSyncConfirmationDelay
 
 	eventRecords = []*clerk.EventRecordWithTime{
 		buildStateEvent(sample, 5, 7),
@@ -578,7 +599,7 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
 			currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)}
 		}
 
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators, false)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -591,7 +612,11 @@ func TestOutOfTurnSigning(t *testing.T) {
 	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
 	fdlimit.Raise(2048)
 
-	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase())
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize}
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
 	chain := init.ethereum.BlockChain()
 	engine := init.ethereum.Engine()
 	_bor := engine.(*bor.Bor)
@@ -626,7 +651,8 @@ func TestOutOfTurnSigning(t *testing.T) {
 	for i := uint64(1); i < spanSize; i++ {
 		// Update the validator set before sprint end (so that it is returned when called for next block)
 		// E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification
-		if i == sprintSize-1 {
+		// Sprint length is 4 for this test
+		if i == chain.Config().Bor.CalculateSprint(i)-1 {
 			currentValidators = heimdallSpan.ValidatorSet.Validators
 
 			// Update the span0's validator set to new validator set. This will be used in verify header when we query
@@ -634,7 +660,7 @@ func TestOutOfTurnSigning(t *testing.T) {
 			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
 			span0.ValidatorSet.Validators = currentValidators
 		}
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, setDifficulty)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, setDifficulty)
 		insertNewBlock(t, chain, block)
 	}
 
@@ -658,7 +684,7 @@ func TestOutOfTurnSigning(t *testing.T) {
 		header.Difficulty = big.NewInt(int64(len(heimdallSpan.ValidatorSet.Validators)) - turn)
 	}
 
-	block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, setParentTime, setDifficulty)
+	block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, false, setParentTime, setDifficulty)
 	_, err := chain.InsertChain([]*types.Block{block})
 	require.Equal(t,
 		bor.BlockTooSoonError{Number: spanSize, Succession: expectedSuccessionNumber},
@@ -723,7 +749,7 @@ func TestSignerNotFound(t *testing.T) {
 		return crypto.Sign(crypto.Keccak256(data), newKey)
 	})
 
-	block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators)
+	block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, false)
 
 	_, err := chain.InsertChain([]*types.Block{block})
 	require.Equal(t,
@@ -1443,7 +1469,7 @@ func TestJaipurFork(t *testing.T) {
 			// stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache.
 			span0.ValidatorSet.Validators = currentValidators
 		}
-		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators)
+		block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false)
 		insertNewBlock(t, chain, block)
 
 		if block.Number().Uint64() == init.genesis.Config.Bor.JaipurBlock.Uint64()-1 {
@@ -1491,3 +1517,359 @@ func testEncodeSigHeader(w io.Writer, header *types.Header, c *params.BorConfig)
 		panic("can't encode: " + err.Error())
 	}
 }
+
+// TestEarlyBlockAnnouncementPostBhilai_Primary tests for different cases of early block announcement
+// acting as a primary block producer. It ensures that consensus handles the header time and
+// block announcement time correctly.
+func TestEarlyBlockAnnouncementPostBhilai_Primary(t *testing.T) {
+	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
+	fdlimit.Raise(2048)
+
+	// Setup forks from genesis block with 2s block time for simplicity
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Timestamp = uint64(time.Now().Unix())
+		gen.Config.Bor.Period = map[string]uint64{"0": 2}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": 16}
+		gen.Config.LondonBlock = common.Big0
+		gen.Config.ShanghaiBlock = common.Big0
+		gen.Config.CancunBlock = common.Big0
+		gen.Config.PragueBlock = common.Big0
+		gen.Config.Bor.JaipurBlock = common.Big0
+		gen.Config.Bor.DelhiBlock = common.Big0
+		gen.Config.Bor.IndoreBlock = common.Big0
+		gen.Config.Bor.BhilaiBlock = common.Big0
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
+
+	chain := init.ethereum.BlockChain()
+	engine := init.ethereum.Engine()
+	_bor := engine.(*bor.Bor)
+	defer _bor.Close()
+
+	span0 := createMockSpan(addr, chain.Config().ChainID.String())
+	_, currentSpan := loadSpanFromFile(t)
+
+	// Create mock heimdall client
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	h := createMockHeimdall(ctrl, &span0, currentSpan)
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
+	_bor.SetHeimdallClient(h)
+
+	block := init.genesis.ToBlock()
+	currentValidators := span0.ValidatorSet.Validators
+
+	spanner := getMockedSpanner(t, currentValidators)
+	_bor.SetSpanner(spanner)
+
+	// Pre-define succession as 0 as all the tests are for primary
+	succession := 0
+	getSuccession := func() int {
+		return succession
+	}
+	updateTime := func(header *types.Header) {
+		// This logic matches with consensus.Prepare function. It's done explicitly here
+		// because other tests aren't designed to use current time and hence might break.
+		if header.Time < uint64(time.Now().Unix()) {
+			header.Time = uint64(time.Now().Unix())
+		} else {
+			if chain.Config().Bor.IsBhilai(header.Number) && getSuccession() == 0 {
+				period := chain.Config().Bor.CalculatePeriod(header.Number.Uint64())
+				startTime := time.Unix(int64(header.Time-period), 0)
+				time.Sleep(time.Until(startTime))
+			}
+		}
+	}
+
+	// Build block 1 normally
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime)
+	i, err := chain.InsertChain([]*types.Block{block})
+	// Block verified and imported successfully
+	require.NoError(t, err, "error inserting block #1")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #1")
+
+	// Case 1: Block announced before header time should be accepted
+	// Block 2
+	// The previous was built early but `updateTime` function will ensure block building
+	// doesn't start before the block's 2s time window.
+	waitingTime := time.Until(time.Unix(int64(block.Time()), 0))
+	// Capture the expected header time based on the logic used in bor consensus
+	headerTime := block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)
+	// Define a max possible delay which is time until header time + waiting time defined above
+	maxDelay := time.Until(time.Unix(int64(headerTime), 0)) + waitingTime
+	// Track time taken to build, and seal (basically announce) the block
+	start := time.Now()
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime)
+	blockAnnouncementTime := time.Since(start)
+	// The building + sealing time should be less than the expected pre-bhilai block building time (~2s)
+	require.LessOrEqual(t, blockAnnouncementTime, maxDelay, fmt.Sprintf("block announcement happened after header time"))
+	// The building + sealing time should be slightly greater than the waiting time
+	require.Greater(t, blockAnnouncementTime, waitingTime, fmt.Sprintf("block announcement time is less than waiting time"))
+	// Block verified and imported successfully
+	i, err = chain.InsertChain([]*types.Block{block})
+	require.NoError(t, err, "error inserting block #2")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #2")
+
+	// Case 2: Delayed block (after header time) should be accepted
+	// Block 3
+	// Wait until header.Time + 1s before building the block
+	headerTime = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)
+	time.Sleep(time.Until(time.Unix(int64(headerTime)+1, 0)))
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime)
+	require.Greater(t, block.Time(), headerTime, "block time should be greated than expected header time")
+	// Block verified and imported successfully
+	i, err = chain.InsertChain([]*types.Block{block})
+	require.NoError(t, err, "error inserting block #3")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #3")
+
+	// Build block 4 normally
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime)
+	i, err = chain.InsertChain([]*types.Block{block})
+	// Block verified and imported successfully
+	require.NoError(t, err, "error inserting block #4")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #4")
+
+	// Case 3: Block announced before it's expected time (header.Time - 2s) should be rejected
+	// Block 5
+	// Use signer to sign block instead of using `bor.Seal` call. This is done to immediately
+	// build the next block instead of waiting for the delay (using bor.Seal will not lead
+	// to block being rejected).
+	updateTimeWithoutSleep := func(header *types.Header) {
+		// This logic matches with consensus.Prepare function. It's done explicitly here
+		// because other tests aren't designed to use current time and hence might break.
+		if header.Time < uint64(time.Now().Unix()) {
+			header.Time = uint64(time.Now().Unix())
+		}
+	}
+	signer, err := hex.DecodeString(privKey)
+	tempBlock := buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTimeWithoutSleep)
+	i, err = chain.InsertChain([]*types.Block{tempBlock})
+	// No error is expected here because block will be added to future chain and is
+	// technically valid (according to insert chain function)
+	require.NoError(t, err, "error inserting block #5")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #5")
+	// Block is invalid according to consensus rules and should return appropriate error
+	err = engine.VerifyHeader(chain, tempBlock.Header())
+	require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #5")
+
+	// Build block 5 again normally
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime)
+	i, err = chain.InsertChain([]*types.Block{block})
+	// Block verified and imported successfully
+	require.NoError(t, err, "error inserting block #5")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #5")
+
+	// Case 4: Block with tweaked header time ahead of expected time should be rejected
+	// Block 6
+	// Set the header time to be 1s earlier than the expected header time
+	setTime := func(header *types.Header) {
+		header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) - 1
+	}
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, setTime)
+	// Consensus verification will fail and this error will float up unlike future block error
+	// as we've tweaked the header time which is not allowed.
+	i, err = chain.InsertChain([]*types.Block{block})
+	require.Equal(t, bor.ErrInvalidTimestamp, err, "incorrect error while inserting block #5")
+	require.Equal(t, 0, i, "incorrect number of blocks inserted while inserting block #5")
+}
+
+// TestEarlyBlockAnnouncementPostBhilai_NonPrimary tests for different cases of early block announcement
+// acting as a non-primary block producer. It ensures that consensus handles the header time and
+// block announcement time correctly.
+func TestEarlyBlockAnnouncementPostBhilai_NonPrimary(t *testing.T) {
+	t.Parallel()
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true)))
+	fdlimit.Raise(2048)
+
+	// Setup forks from genesis block with 2s block time for simplicity
+	updateGenesis := func(gen *core.Genesis) {
+		gen.Timestamp = uint64(time.Now().Unix())
+		gen.Config.Bor.Period = map[string]uint64{"0": 2}
+		gen.Config.Bor.Sprint = map[string]uint64{"0": 16}
+		gen.Config.Bor.ProducerDelay = map[string]uint64{"0": 4}
+		gen.Config.Bor.BackupMultiplier = map[string]uint64{"0": 2}
+		gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128}
+		gen.Config.LondonBlock = common.Big0
+		gen.Config.ShanghaiBlock = common.Big0
+		gen.Config.CancunBlock = common.Big0
+		gen.Config.PragueBlock = common.Big0
+		gen.Config.Bor.JaipurBlock = common.Big0
+		gen.Config.Bor.DelhiBlock = common.Big0
+		gen.Config.Bor.IndoreBlock = common.Big0
+		gen.Config.Bor.BhilaiBlock = common.Big0
+	}
+	init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis)
+
+	chain := init.ethereum.BlockChain()
+	engine := init.ethereum.Engine()
+	_bor := engine.(*bor.Bor)
+	defer _bor.Close()
+
+	// Use 3 validators from the start to allow out-of-turn block production
+	_, span0 := loadSpanFromFile(t)
+	span0.StartBlock = 0
+	span0.EndBlock = 255
+	_, span1 := loadSpanFromFile(t)
+
+	// key2 and addr2 belong to the primary validator, authorize consensus to sign messages
+	engine.(*bor.Bor).Authorize(addr2, func(account accounts.Account, s string, data []byte) ([]byte, error) {
+		return crypto.Sign(crypto.Keccak256(data), key2)
+	})
+
+	// Create mock heimdall client
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	h := createMockHeimdall(ctrl, span0, span1)
+	h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()).
+		Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
+	_bor.SetHeimdallClient(h)
+
+	block := init.genesis.ToBlock()
+	currentValidators := span0.ValidatorSet.Validators
+
+	spanner := getMockedSpanner(t, currentValidators)
+	_bor.SetSpanner(spanner)
+
+	succession := 0
+	getSuccession := func() int {
+		return succession
+	}
+	updateTime := func(header *types.Header) {
+		// This logic matches with consensus.Prepare function. It's done explicitly here
+		// because other tests aren't designed to use current time and hence might break.
+		if header.Time < uint64(time.Now().Unix()) {
+			header.Time = uint64(time.Now().Unix())
+		} else {
+			if chain.Config().Bor.IsBhilai(header.Number) && getSuccession() == 0 {
+				period := chain.Config().Bor.CalculatePeriod(header.Number.Uint64())
+				startTime := time.Unix(int64(header.Time-period), 0)
+				time.Sleep(time.Until(startTime))
+			}
+		}
+	}
+
+	// Build block 1 normally with the primary validator
+	updateDiff := func(header *types.Header) {
+		// We need to explicitly set it otherwise it derives value from
+		// parent block (which is genesis) which we don't want.
+		header.Difficulty = new(big.Int).SetUint64(3)
+	}
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime, updateDiff)
+	i, err := chain.InsertChain([]*types.Block{block})
+	require.NoError(t, err, "error inserting block #1")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #1")
+
+	// Going ahead, all blocks will be built by the tertiary (backup) validator. Authorize consensus
+	// to sign messages on behalf of it's private keys
+	engine.(*bor.Bor).Authorize(addr3, func(account accounts.Account, s string, data []byte) ([]byte, error) {
+		sig, err := crypto.Sign(crypto.Keccak256(data), key3)
+		return sig, err
+	})
+
+	// All blocks from this point will be built by the tertiary validator. Set the succession to 2
+	succession = 2
+
+	// Case 1: Build a block from tertiary validator with header.Time set before block 1's time
+	// As the time in header is invalid, the block should be rejected due to invalid timestamp.
+	// Use signer to sign block instead of using `bor.Seal` call. This is done to immediately
+	// build the next block instead of waiting for the delay.
+	// Block 2
+	signer, _ := hex.DecodeString(privKey3)
+	updateHeader := func(header *types.Header) {
+		header.Difficulty = new(big.Int).SetUint64(1)
+		header.Time = block.Time() - 1
+	}
+	tempBlock := buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader)
+	i, err = chain.InsertChain([]*types.Block{tempBlock})
+	require.Equal(t, bor.ErrInvalidTimestamp, err, "incorrect error while inserting block #2")
+	require.Equal(t, 0, i, "incorrect number of blocks inserted while inserting block #2")
+
+	// Case 2: Build a block from tertiary validator with header.Time set correctly (previous + 6s).
+	// Announce the block early before the previous block's announcement window is over. This should
+	// lead to future block error from consensus.
+	// Block 2 again, build with correct time but announce early
+	updateHeader = func(header *types.Header) {
+		header.Difficulty = new(big.Int).SetUint64(1)
+		// Succession is 2 because of tertiary validator
+		header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)
+	}
+	tempBlock = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader)
+	// Block is invalid according to consensus rules and should return appropriate error
+	// Insert chain would accept the block as future block so we don't attempt calling it.
+	err = engine.VerifyHeader(chain, tempBlock.Header())
+	require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #2")
+
+	// Case 3: Happy case. Build a correct block and ensure the sealing function waits until expected
+	// header time before announcing the block. Non-primary validators can't announce blocks early.
+	var expectedBlockBuildingTime time.Duration
+	updateHeader = func(header *types.Header) {
+		header.Difficulty = new(big.Int).SetUint64(1)
+		header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)
+		// Capture the expected header time based on the logic used in bor consensus
+		expectedBlockBuildingTime = time.Until(time.Unix(int64(header.Time), 0))
+	}
+	// Capture the time taken in block building (mainly sealing due to delay)
+	start := time.Now()
+	block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime, updateHeader)
+	blockAnnouncementTime := time.Since(start)
+	// The building + sealing time should be greater than ideal time (6s for tertiary validator)
+	// as early block announcement is not allowed for non-primary validators.
+	require.GreaterOrEqual(t, blockAnnouncementTime, expectedBlockBuildingTime, fmt.Sprintf("block #2 announcement happened before header time for non-primary validator"))
+	i, err = chain.InsertChain([]*types.Block{block})
+	require.NoError(t, err, "error inserting block #2")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #2")
+
+	// Case 4: Build a block from tertiary validator with correct header time but try to announce it
+	// before it's expected time (i.e. 6s here). Early announcements for non-primary validators
+	// should be rejected with a future block error from consensus.
+	// Block 3 (tertiary)
+	updateHeader = func(header *types.Header) {
+		header.Difficulty = new(big.Int).SetUint64(1)
+		header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)
+	}
+	block = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader)
+
+	// reject if announced early (here: parent block time + 2s)
+	time.Sleep(2 * time.Second)
+	err = engine.VerifyHeader(chain, block.Header())
+	require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #3")
+
+	// reject if announced early (here: parent block time + 4s)
+	time.Sleep(2 * time.Second)
+	err = engine.VerifyHeader(chain, block.Header())
+	require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #3")
+
+	// accept if announced after expected header.Time (here: parent block time + 6s)
+	time.Sleep(2 * time.Second)
+	err = engine.VerifyHeader(chain, block.Header())
+	require.NoError(t, err, "error verifying block #3")
+
+	i, err = chain.InsertChain([]*types.Block{block})
+	require.NoError(t, err, "error inserting block #3")
+	require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #3")
+
+	// Case 5: Build a block from tertiary validator with an incorrect header time (1s before parent block) and
+	// announce it on time. This case is different than case 1 because header time is tweaked by only 1s compared
+	// to 7s in that case. Consensus should reject this block with a too soon error (instead of invalid timestamp
+	// in case 1).
+	updateHeader = func(header *types.Header) {
+		header.Difficulty = new(big.Int).SetUint64(1)
+		header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) - 1
+	}
+	// Capture time to wait until the expected header time before announcing the block
+	timeToWait := time.Until(time.Unix(int64(block.Time()+bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)), 0))
+	block = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader)
+
+	// Wait for expected time + some buffer
+	time.Sleep(timeToWait)
+	time.Sleep(100 * time.Millisecond)
+
+	err = engine.VerifyHeader(chain, block.Header())
+	require.Equal(t,
+		bor.BlockTooSoonError{Number: 4, Succession: 2},
+		*err.(*bor.BlockTooSoonError))
+}

tests/bor/helper.go

@@ -3,6 +3,7 @@
 package bor
 
 import (
+	"bytes"
 	"crypto/ecdsa"
 	"encoding/hex"
 	"encoding/json"
@@ -44,7 +45,9 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/tests/bor/mocks"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/triedb"
 )
 
@@ -58,12 +61,17 @@ var (
 	key2, _ = crypto.HexToECDSA(privKey2)
 	addr2   = crypto.PubkeyToAddress(key2.PublicKey) // 0x9fB29AAc15b9A4B7F17c3385939b007540f4d791
 
+	// This account is secondary validator for 1st span (0-indexed)
+	key3, _ = crypto.HexToECDSA(privKey3)
+	addr3   = crypto.PubkeyToAddress(key3.PublicKey) // 0x96C42C56fdb78294F96B0cFa33c92bed7D75F96a
+
 	keys = []*ecdsa.PrivateKey{key, key2}
 )
 
 const (
 	privKey  = "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"
 	privKey2 = "9b28f36fbd67381120752d6172ecdcf10e06ab2d9a1367aac00cdcd6ac7855d3"
+	privKey3 = "c8deb0bea5c41afe8e37b4d1bd84e31adff11b09c8c96ff4b605003cce067cd9"
 
 	// The genesis for tests was generated with following parameters
 	extraSeal = 65 // Fixed number of extra-data suffix bytes reserved for signer seal
@@ -112,24 +120,25 @@ func setupMiner(t *testing.T, n int, genesis *core.Genesis) ([]*node.Node, []*et
 	return stacks, nodes, enodes
 }
 
-func buildEthereumInstance(t *testing.T, db ethdb.Database) *initializeData {
+func buildEthereumInstance(t *testing.T, db ethdb.Database, updateGenesis ...func(gen *core.Genesis)) *initializeData {
 	genesisData, err := ioutil.ReadFile("./testdata/genesis.json")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
 
 	gen := &core.Genesis{}
-
 	if err := json.Unmarshal(genesisData, gen); err != nil {
 		t.Fatalf("%s", err)
 	}
+	for _, update := range updateGenesis {
+		update(gen)
+	}
 
 	ethConf := &eth.Config{
 		Genesis:     gen,
 		BorLogs:     true,
 		StateScheme: "hash",
 	}
-
 	ethConf.Genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults))
 
 	ethereum := utils.CreateBorEthereum(ethConf)
@@ -159,7 +168,7 @@ func insertNewBlock(t *testing.T, chain *core.BlockChain, block *types.Block) {
 
 type Option func(header *types.Header)
 
-func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, txs []*types.Transaction, currentValidators []*valset.Validator, opts ...Option) *types.Block {
+func buildHeader(t *testing.T, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, currentValidators []*valset.Validator, opts ...Option) *types.Header {
 	t.Helper()
 
 	header := &types.Header{
@@ -174,11 +183,25 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain,
 		signer = getSignerKey(header.Number.Uint64())
 	}
 
+	// Similar to the logic in bor consensus
 	header.Time = parentBlock.Time() + bor.CalcProducerDelay(header.Number.Uint64(), 0, borConfig)
-	header.Extra = make([]byte, 32+65) // vanity + extraSeal
+	// Keeping this causes some e2e tests to fail because they work under certain time assumptions
+	// if header.Time < uint64(time.Now().Unix()) {
+	// 	header.Time = uint64(time.Now().Unix())
+	// }
 
+	// Similar to logic in bor consensus (prepare)
+	header.Extra = make([]byte, 32+65) // vanity + extraSeal
+	if len(header.Extra) < types.ExtraVanityLength {
+		header.Extra = append(header.Extra, bytes.Repeat([]byte{0x00}, types.ExtraVanityLength-len(header.Extra))...)
+	}
+	header.Extra = header.Extra[:types.ExtraVanityLength]
+
+	var isSprintEnd bool
+	if (number+1)%chain.Config().Bor.CalculateSprint(number) == 0 {
+		isSprintEnd = true
+	}
 	isSpanStart := IsSpanStart(number)
-	isSprintEnd := IsSprintEnd(number)
 
 	if isSpanStart {
 		header.Difficulty = new(big.Int).SetInt64(int64(len(currentValidators)))
@@ -187,16 +210,48 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain,
 	if isSprintEnd {
 		sort.Sort(valset.ValidatorsByAddress(currentValidators))
 
-		validatorBytes := make([]byte, len(currentValidators)*validatorHeaderBytesLength)
+		// Extra data is encoded differently after cancun
-		header.Extra = make([]byte, 32+len(validatorBytes)+65) // vanity + validatorBytes + extraSeal
+		if chain.Config().IsCancun(header.Number) {
+			var tempValidatorBytes []byte
+			for _, validator := range currentValidators {
+				tempValidatorBytes = append(tempValidatorBytes, validator.HeaderBytes()...)
+			}
 
-		for i, val := range currentValidators {
+			blockExtraData := &types.BlockExtraData{
-			copy(validatorBytes[i*validatorHeaderBytesLength:], val.HeaderBytes())
+				ValidatorBytes: tempValidatorBytes,
+				TxDependency:   nil,
+			}
+			blockExtraDataBytes, err := rlp.EncodeToBytes(blockExtraData)
+			if err != nil {
+				t.Fatalf("error while encoding block extra data: %v", err)
+			}
+			header.Extra = append(header.Extra, blockExtraDataBytes...)
+		} else {
+			validatorBytes := make([]byte, len(currentValidators)*validatorHeaderBytesLength)
+			header.Extra = make([]byte, 32+len(validatorBytes)+65) // vanity + validatorBytes + extraSeal
+
+			for i, val := range currentValidators {
+				copy(validatorBytes[i*validatorHeaderBytesLength:], val.HeaderBytes())
+			}
+
+			copy(header.Extra[32:], validatorBytes)
+		}
+	} else if chain.Config().IsCancun(header.Number) {
+		blockExtraData := &types.BlockExtraData{
+			ValidatorBytes: nil,
+			TxDependency:   nil,
 		}
 
-		copy(header.Extra[32:], validatorBytes)
+		blockExtraDataBytes, err := rlp.EncodeToBytes(blockExtraData)
+		if err != nil {
+			t.Fatalf("error while encoding block extra data: %v", err)
+		}
+
+		header.Extra = append(header.Extra, blockExtraDataBytes...)
 	}
 
+	header.Extra = append(header.Extra, make([]byte, types.ExtraSealLength)...)
+
 	if chain.Config().IsLondon(header.Number) {
 		header.BaseFee = eip1559.CalcBaseFee(chain.Config(), parentBlock.Header())
 
@@ -210,6 +265,15 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain,
 		opt(header)
 	}
 
+	return header
+}
+
+func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, txs []*types.Transaction, currentValidators []*valset.Validator, skipSealing bool, opts ...Option) *types.Block {
+	t.Helper()
+
+	// Build a new header based on parent block
+	header := buildHeader(t, chain, parentBlock, signer, borConfig, currentValidators, opts...)
+
 	state, err := chain.State()
 	if err != nil {
 		t.Fatalf("%s", err)
@@ -224,7 +288,6 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain,
 	block, err := _bor.FinalizeAndAssemble(chain, b.header, state, &types.Body{
 		Transactions: b.txs,
 	}, b.receipts)
-
 	if err != nil {
 		panic(fmt.Sprintf("error finalizing block: %v", err))
 	}
@@ -241,6 +304,12 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain,
 
 	res := make(chan *types.Block, 1)
 
+	if skipSealing {
+		header := block.Header()
+		sign(t, header, signer, borConfig)
+		return types.NewBlock(header, block.Body(), b.receipts, trie.NewStackTrie(nil))
+	}
+
 	err = _bor.Seal(chain, block, res, nil)
 	if err != nil {
 		// an error case - sign manually

tests/bor/testdata/genesis.json

@@ -13,9 +13,15 @@
     "muirGlacierBlock": 0,
     "berlinBlock": 0,
     "londonBlock": 1,
+    "shanghaiBlock": 3,
+    "cancunBlock": 3,
+    "pragueBlock": 3,
     "bor": {
       "jaipurBlock": 2,
       "delhiBlock" :3,
+      "indoreBlock": 3,
+      "ahmedabadBlock": 3,
+      "bhilaiBlock": 3,
       "period": {
         "0": 1
       },

tests/bor/testdata/genesis_2val.json

@@ -13,9 +13,15 @@
       "muirGlacierBlock": 0,
       "berlinBlock": 0,
       "londonBlock": 1,
+      "shanghaiBlock": 3,
+      "cancunBlock": 3,
+      "pragueBlock": 3,
       "bor": {
         "jaipurBlock": 2,
         "delhiBlock" :3,
+        "indoreBlock": 3,
+        "ahmedabadBlock": 3,
+        "bhilaiBlock": 3,
         "period": {
           "0": 1
         },