diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index adb9e7857b..65496c8a15 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -348,10 +348,24 @@ func (c *Bor) verifyHeader(chain consensus.ChainHeaderReader, header *types.Head } number := header.Number.Uint64() + now := uint64(time.Now().Unix()) - // Don't waste time checking blocks from the future - if header.Time > uint64(time.Now().Unix()) { - return consensus.ErrFutureBlock + // Allow early blocks if Bhilai HF is enabled + if c.config.IsBhilai(header.Number) { + // Don't waste time checking blocks from the future but allow a buffer of block time for + // early block announcements. Note that this is a loose check and would allow early blocks + // from non-primary producer. Such blocks will be rejected later when we know the succession + // number of the signer in the current sprint. + if header.Time-c.config.CalculatePeriod(number) > now { + log.Error("Block announced too early post bhilai", "number", number, "headerTime", header.Time, "now", now) + return consensus.ErrFutureBlock + } + } else { + // Don't waste time checking blocks from the future + if header.Time > now { + log.Error("Block announced too early", "number", number, "headerTime", header.Time, "now", now) + return consensus.ErrFutureBlock + } } if err := validateHeaderExtraField(header.Extra); err != nil { @@ -709,7 +723,16 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header parent = chain.GetHeader(header.ParentHash, number-1) } - if IsBlockOnTime(parent, header, number, succession, c.config) { + // Post Bhilai HF, reject blocks form non-primary producers if they're earlier than the expected time + if c.config.IsBhilai(header.Number) && succession != 0 { + now := uint64(time.Now().Unix()) + if header.Time > now { + log.Error("Block announced too early by non-primary producer post bhilai", "number", number, "headerTime", header.Time, "now", now) + return consensus.ErrFutureBlock + } + } + + if IsBlockEarly(parent, header, number, succession, c.config) { return &BlockTooSoonError{number, succession} } @@ -724,7 +747,9 @@ func (c *Bor) verifySeal(chain consensus.ChainHeaderReader, header *types.Header return nil } -func IsBlockOnTime(parent *types.Header, header *types.Header, number uint64, succession int, cfg *params.BorConfig) bool { +// IsBlockEarly returns true if the header time is earlier than expected (according to consensus rules). This +// can happen if the producer maliciously updates the header time. +func IsBlockEarly(parent *types.Header, header *types.Header, number uint64, succession int, cfg *params.BorConfig) bool { return parent != nil && header.Time < parent.Time+CalcProducerDelay(number, succession, cfg) } @@ -827,6 +852,16 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header) e header.Time = parent.Time + CalcProducerDelay(number, succession, c.config) if header.Time < uint64(time.Now().Unix()) { header.Time = uint64(time.Now().Unix()) + } else { + // For primary validators, wait until the current block production window + // starts. This prevents bor from starting to build next block before time + // as we'd like to wait for new transactions. Although this change doesn't + // need a check for hard fork as it doesn't change any consensus rules, we + // still keep it for safety and testing. + if c.config.IsBhilai(big.NewInt(int64(number))) && succession == 0 { + startTime := time.Unix(int64(header.Time-c.config.CalculatePeriod(number)), 0) + time.Sleep(time.Until(startTime)) + } } return nil @@ -1021,8 +1056,20 @@ func (c *Bor) Seal(chain consensus.ChainHeaderReader, block *types.Block, result return err } + var delay time.Duration + // Sweet, the protocol permits us to sign the block, wait for our time - delay := time.Unix(int64(header.Time), 0).Sub(time.Now()) // nolint: gosimple + if c.config.IsBhilai(header.Number) { + delay = time.Until(time.Unix(int64(header.Time), 0)) // Wait until we reach header time for non-primary validators + if successionNumber == 0 { + // For primary producers, set the delay to `header.Time - block time` instead of `header.Time` + // for early block announcement instead of waiting for full block time. + delay = time.Until(time.Unix(int64(header.Time-c.config.CalculatePeriod(number)), 0)) + } + } else { + delay = time.Until(time.Unix(int64(header.Time), 0)) // Wait until we reach header time + } + // wiggle was already accounted for in header.Time, this is just for logging wiggle := time.Duration(successionNumber) * time.Duration(c.config.CalculateBackupMultiplier(number)) * time.Second diff --git a/tests/bor/bor_test.go b/tests/bor/bor_test.go index 5ac8ccf464..0ed6aa241c 100644 --- a/tests/bor/bor_test.go +++ b/tests/bor/bor_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/bor" "github.com/ethereum/go-ethereum/consensus/bor/clerk" "github.com/ethereum/go-ethereum/consensus/bor/valset" @@ -49,7 +50,6 @@ var ( ) func TestValidatorWentOffline(t *testing.T) { - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) @@ -212,7 +212,6 @@ func TestValidatorWentOffline(t *testing.T) { } func TestForkWithBlockTime(t *testing.T) { - cases := []struct { name string sprint map[string]uint64 @@ -372,7 +371,11 @@ func TestInsertingSpanSizeBlocks(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) - init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) + updateGenesis := func(gen *core.Genesis) { + gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128} + gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize} + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) @@ -399,7 +402,7 @@ func TestInsertingSpanSizeBlocks(t *testing.T) { // Insert sprintSize # of blocks so that span is fetched at the start of a new sprint for i := uint64(1); i <= spanSize; i++ { - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false) insertNewBlock(t, chain, block) } @@ -423,7 +426,12 @@ func TestFetchStateSyncEvents(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) - init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) + stateSyncConfirmationDelay := int64(128) + updateGenesis := func(gen *core.Genesis) { + gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": uint64(stateSyncConfirmationDelay)} + gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize} + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) @@ -451,8 +459,12 @@ func TestFetchStateSyncEvents(t *testing.T) { // Mock state sync events fromID := uint64(1) - // at # sprintSize, events are fetched for [fromID, (block-sprint).Time) - to := int64(chain.GetHeaderByNumber(0).Time) + // at # sprintSize, events are fetched for [fromID, (block-sprint).Time]) + // as indore hf is enabled, we need to consider the stateSyncConfirmationDelay and + // we need to predict the time of 4th block (i.e. the sprint end block) to calculate + // the correct value of to. As per the config, non sprint end primary blocks take + // 1s and sprint end ones take 6s. This leads to 3*1 + 6 = 9s of added time from genesis. + to := int64(chain.GetHeaderByNumber(0).Time) + 9 - stateSyncConfirmationDelay eventCount := 50 sample := getSampleEventRecord(t) @@ -468,11 +480,11 @@ func TestFetchStateSyncEvents(t *testing.T) { currentValidators = res.Result.ValidatorSet.Validators } - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false) insertNewBlock(t, chain, block) } - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators, false) // Validate the state sync transactions set by consensus validateStateSyncEvents(t, eventRecords, chain.GetStateSync()) @@ -493,7 +505,12 @@ func TestFetchStateSyncEvents_2(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) - init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) + stateSyncConfirmationDelay := int64(128) + updateGenesis := func(gen *core.Genesis) { + gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": uint64(stateSyncConfirmationDelay)} + gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize} + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) @@ -517,9 +534,13 @@ func TestFetchStateSyncEvents_2(t *testing.T) { h := createMockHeimdall(ctrl, &span0, &res.Result) // Mock State Sync events - // at # sprintSize, events are fetched for [fromID, (block-sprint).Time) + // at # sprintSize, events are fetched for [fromID, (block-sprint).Time]) + // as indore hf is enabled, we need to consider the stateSyncConfirmationDelay and + // we need to predict the time of 4th block (i.e. the sprint end block) to calculate + // the correct value of to. As per the config, non sprint end primary blocks take + // 1s and sprint end ones take 6s. This leads to 3*1 + 6 = 9s of added time from genesis. fromID := uint64(1) - to := int64(chain.GetHeaderByNumber(0).Time) + to := int64(chain.GetHeaderByNumber(0).Time) + 9 - stateSyncConfirmationDelay sample := getSampleEventRecord(t) // First query will be from [id=1, (block-sprint).Time] @@ -542,7 +563,7 @@ func TestFetchStateSyncEvents_2(t *testing.T) { // Set the current validators from span0 currentValidators := span0.ValidatorSet.Validators for i := uint64(1); i <= sprintSize; i++ { - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false) insertNewBlock(t, chain, block) } @@ -551,9 +572,9 @@ func TestFetchStateSyncEvents_2(t *testing.T) { // state 6 was not written require.Equal(t, uint64(4), lastStateID.Uint64()) - // + // Same calculation for from and to as above fromID = uint64(5) - to = int64(chain.GetHeaderByNumber(sprintSize).Time) + to = int64(chain.GetHeaderByNumber(sprintSize).Time) + 9 - stateSyncConfirmationDelay eventRecords = []*clerk.EventRecordWithTime{ buildStateEvent(sample, 5, 7), @@ -578,7 +599,7 @@ func TestFetchStateSyncEvents_2(t *testing.T) { currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)} } - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators, false) insertNewBlock(t, chain, block) } @@ -591,7 +612,11 @@ func TestOutOfTurnSigning(t *testing.T) { log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) - init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) + updateGenesis := func(gen *core.Genesis) { + gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128} + gen.Config.Bor.Sprint = map[string]uint64{"0": sprintSize} + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) @@ -626,7 +651,8 @@ func TestOutOfTurnSigning(t *testing.T) { for i := uint64(1); i < spanSize; i++ { // Update the validator set before sprint end (so that it is returned when called for next block) // E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification - if i == sprintSize-1 { + // Sprint length is 4 for this test + if i == chain.Config().Bor.CalculateSprint(i)-1 { currentValidators = heimdallSpan.ValidatorSet.Validators // Update the span0's validator set to new validator set. This will be used in verify header when we query @@ -634,7 +660,7 @@ func TestOutOfTurnSigning(t *testing.T) { // stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache. span0.ValidatorSet.Validators = currentValidators } - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, setDifficulty) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, setDifficulty) insertNewBlock(t, chain, block) } @@ -658,7 +684,7 @@ func TestOutOfTurnSigning(t *testing.T) { header.Difficulty = big.NewInt(int64(len(heimdallSpan.ValidatorSet.Validators)) - turn) } - block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, setParentTime, setDifficulty) + block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, false, setParentTime, setDifficulty) _, err := chain.InsertChain([]*types.Block{block}) require.Equal(t, bor.BlockTooSoonError{Number: spanSize, Succession: expectedSuccessionNumber}, @@ -723,7 +749,7 @@ func TestSignerNotFound(t *testing.T) { return crypto.Sign(crypto.Keccak256(data), newKey) }) - block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators) + block = buildNextBlock(t, _bor, chain, block, signerKey, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, false) _, err := chain.InsertChain([]*types.Block{block}) require.Equal(t, @@ -1443,7 +1469,7 @@ func TestJaipurFork(t *testing.T) { // stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache. span0.ValidatorSet.Validators = currentValidators } - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false) insertNewBlock(t, chain, block) if block.Number().Uint64() == init.genesis.Config.Bor.JaipurBlock.Uint64()-1 { @@ -1491,3 +1517,359 @@ func testEncodeSigHeader(w io.Writer, header *types.Header, c *params.BorConfig) panic("can't encode: " + err.Error()) } } + +// TestEarlyBlockAnnouncementPostBhilai_Primary tests for different cases of early block announcement +// acting as a primary block producer. It ensures that consensus handles the header time and +// block announcement time correctly. +func TestEarlyBlockAnnouncementPostBhilai_Primary(t *testing.T) { + t.Parallel() + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + fdlimit.Raise(2048) + + // Setup forks from genesis block with 2s block time for simplicity + updateGenesis := func(gen *core.Genesis) { + gen.Timestamp = uint64(time.Now().Unix()) + gen.Config.Bor.Period = map[string]uint64{"0": 2} + gen.Config.Bor.Sprint = map[string]uint64{"0": 16} + gen.Config.LondonBlock = common.Big0 + gen.Config.ShanghaiBlock = common.Big0 + gen.Config.CancunBlock = common.Big0 + gen.Config.PragueBlock = common.Big0 + gen.Config.Bor.JaipurBlock = common.Big0 + gen.Config.Bor.DelhiBlock = common.Big0 + gen.Config.Bor.IndoreBlock = common.Big0 + gen.Config.Bor.BhilaiBlock = common.Big0 + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) + + chain := init.ethereum.BlockChain() + engine := init.ethereum.Engine() + _bor := engine.(*bor.Bor) + defer _bor.Close() + + span0 := createMockSpan(addr, chain.Config().ChainID.String()) + _, currentSpan := loadSpanFromFile(t) + + // Create mock heimdall client + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + h := createMockHeimdall(ctrl, &span0, currentSpan) + h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()). + Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes() + _bor.SetHeimdallClient(h) + + block := init.genesis.ToBlock() + currentValidators := span0.ValidatorSet.Validators + + spanner := getMockedSpanner(t, currentValidators) + _bor.SetSpanner(spanner) + + // Pre-define succession as 0 as all the tests are for primary + succession := 0 + getSuccession := func() int { + return succession + } + updateTime := func(header *types.Header) { + // This logic matches with consensus.Prepare function. It's done explicitly here + // because other tests aren't designed to use current time and hence might break. + if header.Time < uint64(time.Now().Unix()) { + header.Time = uint64(time.Now().Unix()) + } else { + if chain.Config().Bor.IsBhilai(header.Number) && getSuccession() == 0 { + period := chain.Config().Bor.CalculatePeriod(header.Number.Uint64()) + startTime := time.Unix(int64(header.Time-period), 0) + time.Sleep(time.Until(startTime)) + } + } + } + + // Build block 1 normally + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime) + i, err := chain.InsertChain([]*types.Block{block}) + // Block verified and imported successfully + require.NoError(t, err, "error inserting block #1") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #1") + + // Case 1: Block announced before header time should be accepted + // Block 2 + // The previous was built early but `updateTime` function will ensure block building + // doesn't start before the block's 2s time window. + waitingTime := time.Until(time.Unix(int64(block.Time()), 0)) + // Capture the expected header time based on the logic used in bor consensus + headerTime := block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) + // Define a max possible delay which is time until header time + waiting time defined above + maxDelay := time.Until(time.Unix(int64(headerTime), 0)) + waitingTime + // Track time taken to build, and seal (basically announce) the block + start := time.Now() + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime) + blockAnnouncementTime := time.Since(start) + // The building + sealing time should be less than the expected pre-bhilai block building time (~2s) + require.LessOrEqual(t, blockAnnouncementTime, maxDelay, fmt.Sprintf("block announcement happened after header time")) + // The building + sealing time should be slightly greater than the waiting time + require.Greater(t, blockAnnouncementTime, waitingTime, fmt.Sprintf("block announcement time is less than waiting time")) + // Block verified and imported successfully + i, err = chain.InsertChain([]*types.Block{block}) + require.NoError(t, err, "error inserting block #2") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #2") + + // Case 2: Delayed block (after header time) should be accepted + // Block 3 + // Wait until header.Time + 1s before building the block + headerTime = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) + time.Sleep(time.Until(time.Unix(int64(headerTime)+1, 0))) + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime) + require.Greater(t, block.Time(), headerTime, "block time should be greated than expected header time") + // Block verified and imported successfully + i, err = chain.InsertChain([]*types.Block{block}) + require.NoError(t, err, "error inserting block #3") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #3") + + // Build block 4 normally + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime) + i, err = chain.InsertChain([]*types.Block{block}) + // Block verified and imported successfully + require.NoError(t, err, "error inserting block #4") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #4") + + // Case 3: Block announced before it's expected time (header.Time - 2s) should be rejected + // Block 5 + // Use signer to sign block instead of using `bor.Seal` call. This is done to immediately + // build the next block instead of waiting for the delay (using bor.Seal will not lead + // to block being rejected). + updateTimeWithoutSleep := func(header *types.Header) { + // This logic matches with consensus.Prepare function. It's done explicitly here + // because other tests aren't designed to use current time and hence might break. + if header.Time < uint64(time.Now().Unix()) { + header.Time = uint64(time.Now().Unix()) + } + } + signer, err := hex.DecodeString(privKey) + tempBlock := buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTimeWithoutSleep) + i, err = chain.InsertChain([]*types.Block{tempBlock}) + // No error is expected here because block will be added to future chain and is + // technically valid (according to insert chain function) + require.NoError(t, err, "error inserting block #5") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #5") + // Block is invalid according to consensus rules and should return appropriate error + err = engine.VerifyHeader(chain, tempBlock.Header()) + require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #5") + + // Build block 5 again normally + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime) + i, err = chain.InsertChain([]*types.Block{block}) + // Block verified and imported successfully + require.NoError(t, err, "error inserting block #5") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #5") + + // Case 4: Block with tweaked header time ahead of expected time should be rejected + // Block 6 + // Set the header time to be 1s earlier than the expected header time + setTime := func(header *types.Header) { + header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) - 1 + } + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, setTime) + // Consensus verification will fail and this error will float up unlike future block error + // as we've tweaked the header time which is not allowed. + i, err = chain.InsertChain([]*types.Block{block}) + require.Equal(t, bor.ErrInvalidTimestamp, err, "incorrect error while inserting block #5") + require.Equal(t, 0, i, "incorrect number of blocks inserted while inserting block #5") +} + +// TestEarlyBlockAnnouncementPostBhilai_NonPrimary tests for different cases of early block announcement +// acting as a non-primary block producer. It ensures that consensus handles the header time and +// block announcement time correctly. +func TestEarlyBlockAnnouncementPostBhilai_NonPrimary(t *testing.T) { + t.Parallel() + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + fdlimit.Raise(2048) + + // Setup forks from genesis block with 2s block time for simplicity + updateGenesis := func(gen *core.Genesis) { + gen.Timestamp = uint64(time.Now().Unix()) + gen.Config.Bor.Period = map[string]uint64{"0": 2} + gen.Config.Bor.Sprint = map[string]uint64{"0": 16} + gen.Config.Bor.ProducerDelay = map[string]uint64{"0": 4} + gen.Config.Bor.BackupMultiplier = map[string]uint64{"0": 2} + gen.Config.Bor.StateSyncConfirmationDelay = map[string]uint64{"0": 128} + gen.Config.LondonBlock = common.Big0 + gen.Config.ShanghaiBlock = common.Big0 + gen.Config.CancunBlock = common.Big0 + gen.Config.PragueBlock = common.Big0 + gen.Config.Bor.JaipurBlock = common.Big0 + gen.Config.Bor.DelhiBlock = common.Big0 + gen.Config.Bor.IndoreBlock = common.Big0 + gen.Config.Bor.BhilaiBlock = common.Big0 + } + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase(), updateGenesis) + + chain := init.ethereum.BlockChain() + engine := init.ethereum.Engine() + _bor := engine.(*bor.Bor) + defer _bor.Close() + + // Use 3 validators from the start to allow out-of-turn block production + _, span0 := loadSpanFromFile(t) + span0.StartBlock = 0 + span0.EndBlock = 255 + _, span1 := loadSpanFromFile(t) + + // key2 and addr2 belong to the primary validator, authorize consensus to sign messages + engine.(*bor.Bor).Authorize(addr2, func(account accounts.Account, s string, data []byte) ([]byte, error) { + return crypto.Sign(crypto.Keccak256(data), key2) + }) + + // Create mock heimdall client + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + h := createMockHeimdall(ctrl, span0, span1) + h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()). + Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes() + _bor.SetHeimdallClient(h) + + block := init.genesis.ToBlock() + currentValidators := span0.ValidatorSet.Validators + + spanner := getMockedSpanner(t, currentValidators) + _bor.SetSpanner(spanner) + + succession := 0 + getSuccession := func() int { + return succession + } + updateTime := func(header *types.Header) { + // This logic matches with consensus.Prepare function. It's done explicitly here + // because other tests aren't designed to use current time and hence might break. + if header.Time < uint64(time.Now().Unix()) { + header.Time = uint64(time.Now().Unix()) + } else { + if chain.Config().Bor.IsBhilai(header.Number) && getSuccession() == 0 { + period := chain.Config().Bor.CalculatePeriod(header.Number.Uint64()) + startTime := time.Unix(int64(header.Time-period), 0) + time.Sleep(time.Until(startTime)) + } + } + } + + // Build block 1 normally with the primary validator + updateDiff := func(header *types.Header) { + // We need to explicitly set it otherwise it derives value from + // parent block (which is genesis) which we don't want. + header.Difficulty = new(big.Int).SetUint64(3) + } + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime, updateDiff) + i, err := chain.InsertChain([]*types.Block{block}) + require.NoError(t, err, "error inserting block #1") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #1") + + // Going ahead, all blocks will be built by the tertiary (backup) validator. Authorize consensus + // to sign messages on behalf of it's private keys + engine.(*bor.Bor).Authorize(addr3, func(account accounts.Account, s string, data []byte) ([]byte, error) { + sig, err := crypto.Sign(crypto.Keccak256(data), key3) + return sig, err + }) + + // All blocks from this point will be built by the tertiary validator. Set the succession to 2 + succession = 2 + + // Case 1: Build a block from tertiary validator with header.Time set before block 1's time + // As the time in header is invalid, the block should be rejected due to invalid timestamp. + // Use signer to sign block instead of using `bor.Seal` call. This is done to immediately + // build the next block instead of waiting for the delay. + // Block 2 + signer, _ := hex.DecodeString(privKey3) + updateHeader := func(header *types.Header) { + header.Difficulty = new(big.Int).SetUint64(1) + header.Time = block.Time() - 1 + } + tempBlock := buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader) + i, err = chain.InsertChain([]*types.Block{tempBlock}) + require.Equal(t, bor.ErrInvalidTimestamp, err, "incorrect error while inserting block #2") + require.Equal(t, 0, i, "incorrect number of blocks inserted while inserting block #2") + + // Case 2: Build a block from tertiary validator with header.Time set correctly (previous + 6s). + // Announce the block early before the previous block's announcement window is over. This should + // lead to future block error from consensus. + // Block 2 again, build with correct time but announce early + updateHeader = func(header *types.Header) { + header.Difficulty = new(big.Int).SetUint64(1) + // Succession is 2 because of tertiary validator + header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) + } + tempBlock = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader) + // Block is invalid according to consensus rules and should return appropriate error + // Insert chain would accept the block as future block so we don't attempt calling it. + err = engine.VerifyHeader(chain, tempBlock.Header()) + require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #2") + + // Case 3: Happy case. Build a correct block and ensure the sealing function waits until expected + // header time before announcing the block. Non-primary validators can't announce blocks early. + var expectedBlockBuildingTime time.Duration + updateHeader = func(header *types.Header) { + header.Difficulty = new(big.Int).SetUint64(1) + header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) + // Capture the expected header time based on the logic used in bor consensus + expectedBlockBuildingTime = time.Until(time.Unix(int64(header.Time), 0)) + } + // Capture the time taken in block building (mainly sealing due to delay) + start := time.Now() + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, false, updateTime, updateHeader) + blockAnnouncementTime := time.Since(start) + // The building + sealing time should be greater than ideal time (6s for tertiary validator) + // as early block announcement is not allowed for non-primary validators. + require.GreaterOrEqual(t, blockAnnouncementTime, expectedBlockBuildingTime, fmt.Sprintf("block #2 announcement happened before header time for non-primary validator")) + i, err = chain.InsertChain([]*types.Block{block}) + require.NoError(t, err, "error inserting block #2") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #2") + + // Case 4: Build a block from tertiary validator with correct header time but try to announce it + // before it's expected time (i.e. 6s here). Early announcements for non-primary validators + // should be rejected with a future block error from consensus. + // Block 3 (tertiary) + updateHeader = func(header *types.Header) { + header.Difficulty = new(big.Int).SetUint64(1) + header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) + } + block = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader) + + // reject if announced early (here: parent block time + 2s) + time.Sleep(2 * time.Second) + err = engine.VerifyHeader(chain, block.Header()) + require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #3") + + // reject if announced early (here: parent block time + 4s) + time.Sleep(2 * time.Second) + err = engine.VerifyHeader(chain, block.Header()) + require.ErrorIs(t, err, consensus.ErrFutureBlock, "incorrect error while verifying block #3") + + // accept if announced after expected header.Time (here: parent block time + 6s) + time.Sleep(2 * time.Second) + err = engine.VerifyHeader(chain, block.Header()) + require.NoError(t, err, "error verifying block #3") + + i, err = chain.InsertChain([]*types.Block{block}) + require.NoError(t, err, "error inserting block #3") + require.Equal(t, 1, i, "incorrect number of blocks inserted while inserting block #3") + + // Case 5: Build a block from tertiary validator with an incorrect header time (1s before parent block) and + // announce it on time. This case is different than case 1 because header time is tweaked by only 1s compared + // to 7s in that case. Consensus should reject this block with a too soon error (instead of invalid timestamp + // in case 1). + updateHeader = func(header *types.Header) { + header.Difficulty = new(big.Int).SetUint64(1) + header.Time = block.Time() + bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor) - 1 + } + // Capture time to wait until the expected header time before announcing the block + timeToWait := time.Until(time.Unix(int64(block.Time()+bor.CalcProducerDelay(block.NumberU64(), getSuccession(), init.genesis.Config.Bor)), 0)) + block = buildNextBlock(t, _bor, chain, block, signer, init.genesis.Config.Bor, nil, currentValidators, true, updateTime, updateHeader) + + // Wait for expected time + some buffer + time.Sleep(timeToWait) + time.Sleep(100 * time.Millisecond) + + err = engine.VerifyHeader(chain, block.Header()) + require.Equal(t, + bor.BlockTooSoonError{Number: 4, Succession: 2}, + *err.(*bor.BlockTooSoonError)) +} diff --git a/tests/bor/helper.go b/tests/bor/helper.go index 99db347f8f..fd5a8dc499 100644 --- a/tests/bor/helper.go +++ b/tests/bor/helper.go @@ -3,6 +3,7 @@ package bor import ( + "bytes" "crypto/ecdsa" "encoding/hex" "encoding/json" @@ -44,7 +45,9 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests/bor/mocks" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" ) @@ -58,12 +61,17 @@ var ( key2, _ = crypto.HexToECDSA(privKey2) addr2 = crypto.PubkeyToAddress(key2.PublicKey) // 0x9fB29AAc15b9A4B7F17c3385939b007540f4d791 + // This account is secondary validator for 1st span (0-indexed) + key3, _ = crypto.HexToECDSA(privKey3) + addr3 = crypto.PubkeyToAddress(key3.PublicKey) // 0x96C42C56fdb78294F96B0cFa33c92bed7D75F96a + keys = []*ecdsa.PrivateKey{key, key2} ) const ( privKey = "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291" privKey2 = "9b28f36fbd67381120752d6172ecdcf10e06ab2d9a1367aac00cdcd6ac7855d3" + privKey3 = "c8deb0bea5c41afe8e37b4d1bd84e31adff11b09c8c96ff4b605003cce067cd9" // The genesis for tests was generated with following parameters extraSeal = 65 // Fixed number of extra-data suffix bytes reserved for signer seal @@ -112,24 +120,25 @@ func setupMiner(t *testing.T, n int, genesis *core.Genesis) ([]*node.Node, []*et return stacks, nodes, enodes } -func buildEthereumInstance(t *testing.T, db ethdb.Database) *initializeData { +func buildEthereumInstance(t *testing.T, db ethdb.Database, updateGenesis ...func(gen *core.Genesis)) *initializeData { genesisData, err := ioutil.ReadFile("./testdata/genesis.json") if err != nil { t.Fatalf("%s", err) } gen := &core.Genesis{} - if err := json.Unmarshal(genesisData, gen); err != nil { t.Fatalf("%s", err) } + for _, update := range updateGenesis { + update(gen) + } ethConf := ð.Config{ Genesis: gen, BorLogs: true, StateScheme: "hash", } - ethConf.Genesis.MustCommit(db, triedb.NewDatabase(db, triedb.HashDefaults)) ethereum := utils.CreateBorEthereum(ethConf) @@ -159,7 +168,7 @@ func insertNewBlock(t *testing.T, chain *core.BlockChain, block *types.Block) { type Option func(header *types.Header) -func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, txs []*types.Transaction, currentValidators []*valset.Validator, opts ...Option) *types.Block { +func buildHeader(t *testing.T, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, currentValidators []*valset.Validator, opts ...Option) *types.Header { t.Helper() header := &types.Header{ @@ -174,11 +183,25 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, signer = getSignerKey(header.Number.Uint64()) } + // Similar to the logic in bor consensus header.Time = parentBlock.Time() + bor.CalcProducerDelay(header.Number.Uint64(), 0, borConfig) - header.Extra = make([]byte, 32+65) // vanity + extraSeal + // Keeping this causes some e2e tests to fail because they work under certain time assumptions + // if header.Time < uint64(time.Now().Unix()) { + // header.Time = uint64(time.Now().Unix()) + // } + // Similar to logic in bor consensus (prepare) + header.Extra = make([]byte, 32+65) // vanity + extraSeal + if len(header.Extra) < types.ExtraVanityLength { + header.Extra = append(header.Extra, bytes.Repeat([]byte{0x00}, types.ExtraVanityLength-len(header.Extra))...) + } + header.Extra = header.Extra[:types.ExtraVanityLength] + + var isSprintEnd bool + if (number+1)%chain.Config().Bor.CalculateSprint(number) == 0 { + isSprintEnd = true + } isSpanStart := IsSpanStart(number) - isSprintEnd := IsSprintEnd(number) if isSpanStart { header.Difficulty = new(big.Int).SetInt64(int64(len(currentValidators))) @@ -187,16 +210,48 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, if isSprintEnd { sort.Sort(valset.ValidatorsByAddress(currentValidators)) - validatorBytes := make([]byte, len(currentValidators)*validatorHeaderBytesLength) - header.Extra = make([]byte, 32+len(validatorBytes)+65) // vanity + validatorBytes + extraSeal + // Extra data is encoded differently after cancun + if chain.Config().IsCancun(header.Number) { + var tempValidatorBytes []byte + for _, validator := range currentValidators { + tempValidatorBytes = append(tempValidatorBytes, validator.HeaderBytes()...) + } - for i, val := range currentValidators { - copy(validatorBytes[i*validatorHeaderBytesLength:], val.HeaderBytes()) + blockExtraData := &types.BlockExtraData{ + ValidatorBytes: tempValidatorBytes, + TxDependency: nil, + } + blockExtraDataBytes, err := rlp.EncodeToBytes(blockExtraData) + if err != nil { + t.Fatalf("error while encoding block extra data: %v", err) + } + header.Extra = append(header.Extra, blockExtraDataBytes...) + } else { + validatorBytes := make([]byte, len(currentValidators)*validatorHeaderBytesLength) + header.Extra = make([]byte, 32+len(validatorBytes)+65) // vanity + validatorBytes + extraSeal + + for i, val := range currentValidators { + copy(validatorBytes[i*validatorHeaderBytesLength:], val.HeaderBytes()) + } + + copy(header.Extra[32:], validatorBytes) + } + } else if chain.Config().IsCancun(header.Number) { + blockExtraData := &types.BlockExtraData{ + ValidatorBytes: nil, + TxDependency: nil, } - copy(header.Extra[32:], validatorBytes) + blockExtraDataBytes, err := rlp.EncodeToBytes(blockExtraData) + if err != nil { + t.Fatalf("error while encoding block extra data: %v", err) + } + + header.Extra = append(header.Extra, blockExtraDataBytes...) } + header.Extra = append(header.Extra, make([]byte, types.ExtraSealLength)...) + if chain.Config().IsLondon(header.Number) { header.BaseFee = eip1559.CalcBaseFee(chain.Config(), parentBlock.Header()) @@ -210,6 +265,15 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, opt(header) } + return header +} + +func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, parentBlock *types.Block, signer []byte, borConfig *params.BorConfig, txs []*types.Transaction, currentValidators []*valset.Validator, skipSealing bool, opts ...Option) *types.Block { + t.Helper() + + // Build a new header based on parent block + header := buildHeader(t, chain, parentBlock, signer, borConfig, currentValidators, opts...) + state, err := chain.State() if err != nil { t.Fatalf("%s", err) @@ -224,7 +288,6 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, block, err := _bor.FinalizeAndAssemble(chain, b.header, state, &types.Body{ Transactions: b.txs, }, b.receipts) - if err != nil { panic(fmt.Sprintf("error finalizing block: %v", err)) } @@ -241,6 +304,12 @@ func buildNextBlock(t *testing.T, _bor consensus.Engine, chain *core.BlockChain, res := make(chan *types.Block, 1) + if skipSealing { + header := block.Header() + sign(t, header, signer, borConfig) + return types.NewBlock(header, block.Body(), b.receipts, trie.NewStackTrie(nil)) + } + err = _bor.Seal(chain, block, res, nil) if err != nil { // an error case - sign manually diff --git a/tests/bor/testdata/genesis.json b/tests/bor/testdata/genesis.json index cbae055a0a..381d7f52f8 100644 --- a/tests/bor/testdata/genesis.json +++ b/tests/bor/testdata/genesis.json @@ -13,9 +13,15 @@ "muirGlacierBlock": 0, "berlinBlock": 0, "londonBlock": 1, + "shanghaiBlock": 3, + "cancunBlock": 3, + "pragueBlock": 3, "bor": { "jaipurBlock": 2, "delhiBlock" :3, + "indoreBlock": 3, + "ahmedabadBlock": 3, + "bhilaiBlock": 3, "period": { "0": 1 }, diff --git a/tests/bor/testdata/genesis_2val.json b/tests/bor/testdata/genesis_2val.json index 5ce5fbcd4e..52ba111287 100644 --- a/tests/bor/testdata/genesis_2val.json +++ b/tests/bor/testdata/genesis_2val.json @@ -13,9 +13,15 @@ "muirGlacierBlock": 0, "berlinBlock": 0, "londonBlock": 1, + "shanghaiBlock": 3, + "cancunBlock": 3, + "pragueBlock": 3, "bor": { "jaipurBlock": 2, "delhiBlock" :3, + "indoreBlock": 3, + "ahmedabadBlock": 3, + "bhilaiBlock": 3, "period": { "0": 1 },