Xin 147 initial for both first v2 block and further, also introduce getExtraField function (#64)

* refactor initial and introduce getExtraField function

* add test for initial

* refactor snapshot

* initial first snapshot only

consensus/XDPoS/XDPoS.go

@@ -63,8 +63,6 @@ type XDPoS struct {
 	// The exact consensus engine with different versions
 	EngineV1 *engine_v1.XDPoS_v1
 	EngineV2 *engine_v2.XDPoS_v2
-
-	isV2Initilised bool
 }
 
 // New creates a XDPoS delegated-proof-of-stake consensus engine with the initial
@@ -92,7 +90,6 @@ func New(config *params.XDPoSConfig, db ethdb.Database) *XDPoS {
 		signingTxsCache: signingTxsCache,
 		EngineV1:        engine_v1.New(config, db),
 		EngineV2:        engine_v2.New(config, db, waitPeriodCh),
-		isV2Initilised:  false,
 	}
 }
 
@@ -317,26 +314,6 @@ func (x *XDPoS) GetMasternodesByNumber(chain consensus.ChainReader, blockNumber
 }
 
 func (x *XDPoS) YourTurn(chain consensus.ChainReader, parent *types.Header, signer common.Address) (bool, error) {
-	if x.config.V2.SwitchBlock != nil && parent.Number.Cmp(x.config.V2.SwitchBlock) != -1 {
-		if parent.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
-			err := x.initialV2FromLastV1(chain, parent)
-			if err != nil {
-				log.Error("[YourTurn] Error while initilising first v2 block from the last v1 block", "ParentBlockHash", parent.Hash(), "Error", err)
-				return false, err
-			}
-			x.isV2Initilised = true
-		} else if parent.Number.Cmp(x.config.V2.SwitchBlock) == 1 && !x.isV2Initilised { // TODO: XIN-147, temporary solution for now
-			log.Info("[YourTurn] Initilising v2 after sync or restarted", "currentBlockNum", chain.CurrentHeader().Number, "currentBlockHash", chain.CurrentHeader().Hash())
-			lastv1BlockHeader := chain.GetHeaderByNumber(x.config.V2.SwitchBlock.Uint64())
-			err := x.initialV2FromLastV1(chain, lastv1BlockHeader)
-			if err != nil {
-				log.Error("[YourTurn] Temporary solution! Error when initialise v2", "lastv1BlockHeader", lastv1BlockHeader.Hash(), "Error", err)
-				return false, err
-			}
-			x.isV2Initilised = true
-		}
-
-	}
 	switch x.config.BlockConsensusVersion(big.NewInt(parent.Number.Int64() + 1)) {
 	case params.ConsensusEngineVersion2:
 		return x.EngineV2.YourTurn(chain, parent, signer)
@@ -353,6 +330,7 @@ func (x *XDPoS) GetValidator(creator common.Address, chain consensus.ChainReader
 }
 
 func (x *XDPoS) UpdateMasternodes(chain consensus.ChainReader, header *types.Header, ms []utils.Masternode) error {
+	// fmt.Println("UpdateMasternodes")
 	switch x.config.BlockConsensusVersion(header.Number) {
 	case params.ConsensusEngineVersion2:
 		return x.EngineV2.UpdateMasternodes(chain, header, ms)
@@ -509,15 +487,3 @@ func (x *XDPoS) CacheSigningTxs(hash common.Hash, txs []*types.Transaction) []*t
 func (x *XDPoS) GetCachedSigningTxs(hash common.Hash) (interface{}, bool) {
 	return x.signingTxsCache.Get(hash)
 }
-
-// V2 specific helper function to initilise consensus engine variables
-func (x *XDPoS) initialV2FromLastV1(chain consensus.ChainReader, header *types.Header) error {
-	checkpointBlockNumber := header.Number.Uint64() - header.Number.Uint64()%x.config.Epoch
-	checkpointHeader := chain.GetHeaderByNumber(checkpointBlockNumber)
-	masternodes := x.EngineV1.GetMasternodesFromCheckpointHeader(checkpointHeader)
-	err := x.EngineV2.Initial(chain, masternodes)
-	if err != nil {
-		return err
-	}
-	return nil
-}

consensus/XDPoS/engines/engine_v2/engine.go

@@ -28,8 +28,9 @@ import (
 )
 
 type XDPoS_v2 struct {
-	config *params.XDPoSConfig // Consensus engine configuration parameters
+	config       *params.XDPoSConfig // Consensus engine configuration parameters
-	db     ethdb.Database      // Database to store and retrieve snapshot checkpoints
+	db           ethdb.Database      // Database to store and retrieve snapshot checkpoints
+	isInitilised bool                // status of v2 variables
 
 	snapshots       *lru.ARCCache // Snapshots for gap block
 	signatures      *lru.ARCCache // Signatures of recent blocks to speed up mining
@@ -74,8 +75,10 @@ func New(config *params.XDPoSConfig, db ethdb.Database, waitPeriodCh chan int) *
 
 	votePool := utils.NewPool(config.V2.CertThreshold)
 	engine := &XDPoS_v2{
-		config:     config,
+		config:       config,
-		db:         db,
+		db:           db,
+		isInitilised: false,
+
 		signatures: signatures,
 
 		verifiedHeaders: verifiedHeaders,
@@ -122,7 +125,7 @@ func (x *XDPoS_v2) SignHash(header *types.Header) (hash common.Hash) {
 	return sigHash(header)
 }
 
-func (x *XDPoS_v2) Initial(chain consensus.ChainReader, masternodes []common.Address) error {
+func (x *XDPoS_v2) Initial(chain consensus.ChainReader, header *types.Header) error {
 	log.Info("[Initial] initial v2 related parameters")
 
 	if x.highestQuorumCert.ProposedBlockInfo.Hash != (common.Hash{}) { // already initialized
@@ -130,36 +133,59 @@ func (x *XDPoS_v2) Initial(chain consensus.ChainReader, masternodes []common.Add
 		return nil
 	}
 
-	x.lock.Lock()
+	var quorumCert *utils.QuorumCert
-	defer x.lock.Unlock()
+	var err error
-	// Check header if it is the first consensus v2 block, if so, assign initial values to current round and highestQC
 
-	log.Info("[Initial] highest QC for consensus v2 first block")
+	if header.Number.Int64() == x.config.V2.SwitchBlock.Int64() {
-	// Generate new parent blockInfo and put it into QC
+		log.Info("[Initial] highest QC for consensus v2 first block")
-	// TODO: XIN-147 to initilise V2 engine in a more dynamic way
+		blockInfo := &utils.BlockInfo{
-	firstV2BlockHeader := chain.GetHeaderByNumber(x.config.V2.SwitchBlock.Uint64())
+			Hash:   header.Hash(),
-	blockInfo := &utils.BlockInfo{
+			Round:  utils.Round(0),
-		Hash:   firstV2BlockHeader.Hash(),
+			Number: header.Number,
-		Round:  utils.Round(0),
+		}
-		Number: firstV2BlockHeader.Number,
+		quorumCert = &utils.QuorumCert{
+			ProposedBlockInfo: blockInfo,
+			Signatures:        nil,
+		}
+
+		// can not call processQC because round is equal to default
+		x.currentRound = 1
+		x.highestQuorumCert = quorumCert
+
+	} else {
+		log.Info("[Initial] highest QC from current header")
+		quorumCert, _, _, err = x.getExtraFields(header)
+		if err != nil {
+			return err
+		}
+		err = x.processQC(chain, quorumCert)
+		if err != nil {
+			return err
+		}
 	}
-	quorumCert := &utils.QuorumCert{
-		ProposedBlockInfo: blockInfo,
-		Signatures:        nil,
-	}
-	x.currentRound = 1
-	x.highestQuorumCert = quorumCert
 
-	// Initial snapshot
+	// Initial first v2 snapshot
-	lastGapNum := firstV2BlockHeader.Number.Uint64() - firstV2BlockHeader.Number.Uint64()%x.config.Epoch - x.config.Gap
+	if header.Number.Uint64() < x.config.V2.SwitchBlock.Uint64()+x.config.Gap {
-	lastGapHeader := chain.GetHeaderByNumber(lastGapNum)
 
-	snap := newSnapshot(lastGapNum, lastGapHeader.Hash(), x.currentRound, x.highestQuorumCert, masternodes)
+		checkpointBlockNumber := header.Number.Uint64() - header.Number.Uint64()%x.config.Epoch
-	x.snapshots.Add(snap.Hash, snap)
+		checkpointHeader := chain.GetHeaderByNumber(checkpointBlockNumber)
-	err := storeSnapshot(snap, x.db)
+
-	if err != nil {
+		lastGapNum := checkpointBlockNumber - x.config.Gap
-		log.Error("[Initial] Error while storo snapshot", "error", err)
+		lastGapHeader := chain.GetHeaderByNumber(lastGapNum)
-		return err
+
+		log.Info("[Initial] init first snapshot")
+		_, _, masternodes, err := x.getExtraFields(checkpointHeader)
+		if err != nil {
+			log.Error("[Initial] Error while get masternodes", "error", err)
+			return err
+		}
+		snap := newSnapshot(lastGapNum, lastGapHeader.Hash(), masternodes)
+		x.snapshots.Add(snap.Hash, snap)
+		err = storeSnapshot(snap, x.db)
+		if err != nil {
+			log.Error("[Initial] Error while store snapshot", "error", err)
+			return err
+		}
 	}
 
 	// Initial timeout
@@ -173,6 +199,7 @@ func (x *XDPoS_v2) Initial(chain consensus.ChainReader, masternodes []common.Add
 	x.timeoutWorker.Reset(chain)
 
 	log.Info("[Initial] finish initialisation")
+
 	return nil
 }
 
@@ -186,6 +213,7 @@ func (x *XDPoS_v2) Prepare(chain consensus.ChainReader, header *types.Header) er
 	x.lock.RUnlock()
 
 	if header.ParentHash != highestQC.ProposedBlockInfo.Hash {
+		fmt.Println("[Prepare] parent hash and QC hash does not match", "blockNum", header.Number, "parentHash", header.ParentHash, "QCHash", highestQC.ProposedBlockInfo.Hash, "QCNumber", highestQC.ProposedBlockInfo.Number)
 		log.Warn("[Prepare] parent hash and QC hash does not match", "blockNum", header.Number, "parentHash", header.ParentHash, "QCHash", highestQC.ProposedBlockInfo.Hash, "QCNumber", highestQC.ProposedBlockInfo.Number)
 		return consensus.ErrNotReadyToPropose
 	}
@@ -303,33 +331,12 @@ func (x *XDPoS_v2) Seal(chain consensus.ChainReader, block *types.Block, stop <-
 	if number == 0 {
 		return nil, utils.ErrUnknownBlock
 	}
-	// For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing)
+
-	// checkpoint blocks have no tx
-	isEpochSwitch, _, err := x.IsEpochSwitch(header)
-	if err != nil {
-		log.Error("[Seal] Error while checking whether header is a epoch switch during sealing", "Header", header)
-	}
-	if x.config.Period == 0 && len(block.Transactions()) == 0 && !isEpochSwitch {
-		return nil, utils.ErrWaitTransactions
-	}
 	// Don't hold the signer fields for the entire sealing procedure
 	x.signLock.RLock()
 	signer, signFn := x.signer, x.signFn
 	x.signLock.RUnlock()
 
-	// Bail out if we're unauthorized to sign a block
-	masternodes := x.GetMasternodes(chain, header)
-	valid := false
-	for _, m := range masternodes {
-		if m == signer {
-			valid = true
-			break
-		}
-	}
-	if !valid {
-		return nil, utils.ErrUnauthorized
-	}
-
 	select {
 	case <-stop:
 		return nil, nil
@@ -364,6 +371,15 @@ func (x *XDPoS_v2) YourTurn(chain consensus.ChainReader, parent *types.Header, s
 	x.lock.RLock()
 	defer x.lock.RUnlock()
 
+	if !x.isInitilised {
+		err := x.Initial(chain, parent)
+		if err != nil {
+			log.Error("[YourTurn] Error while initialising last v2 variables", "ParentBlockHash", parent.Hash(), "Error", err)
+			return false, err
+		}
+		x.isInitilised = true
+	}
+
 	waitedTime := time.Now().Unix() - parent.Time.Int64()
 	if waitedTime < int64(x.config.V2.MinePeriod) {
 		log.Trace("[YourTurn] wait after mine period", "minePeriod", x.config.V2.MinePeriod, "waitedTime", waitedTime)
@@ -379,13 +395,12 @@ func (x *XDPoS_v2) YourTurn(chain consensus.ChainReader, parent *types.Header, s
 	var masterNodes []common.Address
 	if isEpochSwitch {
 		if x.config.V2.SwitchBlock.Cmp(parent.Number) == 0 {
-			snap, err := x.getSnapshot(chain, x.config.V2.SwitchBlock.Uint64(), false)
+			// the initial master nodes of v1->v2 switch contains penalties node
+			_, _, masterNodes, err = x.getExtraFields(parent)
 			if err != nil {
 				log.Error("[YourTurn] Cannot find snapshot at gap num of last V1", "err", err, "number", x.config.V2.SwitchBlock.Uint64())
 				return false, err
 			}
-			// the initial master nodes of v1->v2 switch contains penalties node
-			masterNodes = snap.NextEpochMasterNodes
 		} else {
 			masterNodes, _, err = x.calcMasternodes(chain, big.NewInt(0).Add(parent.Number, big.NewInt(1)), parent.Hash())
 			if err != nil {
@@ -400,36 +415,38 @@ func (x *XDPoS_v2) YourTurn(chain consensus.ChainReader, parent *types.Header, s
 
 	if len(masterNodes) == 0 {
 		log.Error("[YourTurn] Fail to find any master nodes from current block round epoch", "Hash", parent.Hash(), "CurrentRound", round, "Number", parent.Number)
-		return false, errors.New("Masternodes not found")
+		return false, errors.New("masternodes not found")
 	}
-	leaderIndex := uint64(round) % x.config.Epoch % uint64(len(masterNodes))
 
 	curIndex := utils.Position(masterNodes, signer)
-	if signer == x.signer {
+	if curIndex == -1 {
-		log.Debug("[YourTurn] masterNodes cycle info", "number of masternodes", len(masterNodes), "current", signer, "position", curIndex, "parentBlock", parent)
+		log.Debug("[YourTurn] Not authorised signer", "MN", masterNodes, "Hash", parent.Hash(), "signer", signer)
+		return false, nil
 	}
+
 	for i, s := range masterNodes {
 		log.Debug("[YourTurn] Masternode:", "index", i, "address", s.String(), "parentBlockNum", parent.Number)
 	}
 
-	if masterNodes[leaderIndex] == signer {
+	leaderIndex := uint64(round) % x.config.Epoch % uint64(len(masterNodes))
-		return true, nil
+	if masterNodes[leaderIndex] != signer {
+		log.Debug("[YourTurn] Not my turn", "curIndex", curIndex, "leaderIndex", leaderIndex, "Hash", parent.Hash(), "masterNodes[leaderIndex]", masterNodes[leaderIndex], "signer", signer)
+		return false, nil
 	}
-	log.Warn("[YourTurn] Not authorised signer", "signer", signer, "MN", masterNodes, "Hash", parent.Hash().Hex(), "masterNodes[leaderIndex]", masterNodes[leaderIndex], "signer", signer)
+
-	return false, nil
+	return true, nil
 }
 
 func (x *XDPoS_v2) IsAuthorisedAddress(chain consensus.ChainReader, header *types.Header, address common.Address) bool {
 	x.lock.RLock()
 	defer x.lock.RUnlock()
 
-	var extraField utils.ExtraFields_v2
+	_, round, _, err := x.getExtraFields(header)
-	err := utils.DecodeBytesExtraFields(header.Extra, &extraField)
 	if err != nil {
 		log.Error("[IsAuthorisedAddress] Fail to decode v2 extra data", "Hash", header.Hash().Hex(), "Extra", header.Extra, "Error", err)
 		return false
 	}
-	blockRound := extraField.Round
+	blockRound := round
 
 	masterNodes := x.GetMasternodes(chain, header)
 
@@ -483,7 +500,7 @@ func (x *XDPoS_v2) getSnapshot(chain consensus.ChainReader, number uint64, isGap
 		return snap, nil
 	}
 	// If an on-disk checkpoint snapshot can be found, use that
-	snap, err := loadSnapshot(x.signatures, x.db, gapBlockHash)
+	snap, err := loadSnapshot(x.db, gapBlockHash)
 	if err != nil {
 		log.Error("Cannot find snapshot from last gap block", "err", err, "number", gapBlockNum, "hash", gapBlockHash)
 		return nil, err
@@ -504,7 +521,7 @@ func (x *XDPoS_v2) UpdateMasternodes(chain consensus.ChainReader, header *types.
 	}
 
 	x.lock.RLock()
-	snap := newSnapshot(number, header.Hash(), x.currentRound, x.highestQuorumCert, masterNodes)
+	snap := newSnapshot(number, header.Hash(), masterNodes)
 	x.lock.RUnlock()
 
 	err := storeSnapshot(snap, x.db)
@@ -572,12 +589,12 @@ func (x *XDPoS_v2) verifyHeader(chain consensus.ChainReader, header *types.Heade
 	}
 
 	// Verify this is truely a v2 block first
-	var decodedExtraField utils.ExtraFields_v2
+
-	err := utils.DecodeBytesExtraFields(header.Extra, &decodedExtraField)
+	quorumCert, _, _, err := x.getExtraFields(header)
 	if err != nil {
 		return utils.ErrInvalidV2Extra
 	}
-	quorumCert := decodedExtraField.QuorumCert
+
 	err = x.verifyQC(chain, quorumCert)
 	if err != nil {
 		log.Warn("[verifyHeader] fail to verify QC", "QCNumber", quorumCert.ProposedBlockInfo.Number, "QCsigLength", len(quorumCert.Signatures))
@@ -742,8 +759,9 @@ func (x *XDPoS_v2) voteHandler(chain consensus.ChainReader, voteMsg *utils.Vote)
 
 	// Collect vote
 	thresholdReached, numberOfVotesInPool, pooledVotes := x.votePool.Add(voteMsg)
+	log.Info("[voteHandler] collect votes", "number", numberOfVotesInPool)
 	if thresholdReached {
-		log.Info(fmt.Sprintf("Vote pool threashold reached: %v, number of items in the pool: %v", thresholdReached, numberOfVotesInPool))
+		log.Info(fmt.Sprintf("[voteHandler] Vote pool threashold reached: %v, number of items in the pool: %v", thresholdReached, numberOfVotesInPool))
 
 		// Check if the block already exist, otherwise we try luck with the next vote
 		proposedBlockHeader := chain.GetHeaderByHash(voteMsg.ProposedBlockInfo.Hash)
@@ -868,6 +886,8 @@ func (x *XDPoS_v2) timeoutHandler(blockChainReader consensus.ChainReader, timeou
 	}
 	// Collect timeout, generate TC
 	isThresholdReached, numberOfTimeoutsInPool, pooledTimeouts := x.timeoutPool.Add(timeout)
+	log.Info("[timeoutHandler] collect timeout", "number", numberOfTimeoutsInPool)
+
 	// Threshold reached
 	if isThresholdReached {
 		log.Info(fmt.Sprintf("Timeout pool threashold reached: %v, number of items in the pool: %v", isThresholdReached, numberOfTimeoutsInPool))
@@ -928,13 +948,10 @@ func (x *XDPoS_v2) ProposedBlockHandler(chain consensus.ChainReader, blockHeader
 		5. sendVote()
 	*/
 	// Get QC and Round from Extra
-	var decodedExtraField utils.ExtraFields_v2
+	quorumCert, round, _, err := x.getExtraFields(blockHeader)
-	err := utils.DecodeBytesExtraFields(blockHeader.Extra, &decodedExtraField)
 	if err != nil {
 		return err
 	}
-	quorumCert := decodedExtraField.QuorumCert
-	round := decodedExtraField.Round
 
 	err = x.verifyQC(chain, quorumCert)
 	if err != nil {
@@ -1001,15 +1018,15 @@ func (x *XDPoS_v2) VerifyBlockInfo(blockChainReader consensus.ChainReader, block
 		return nil
 	}
 	// Check round
-	var decodedExtraField utils.ExtraFields_v2
+
-	err := utils.DecodeBytesExtraFields(blockHeader.Extra, &decodedExtraField)
+	_, round, _, err := x.getExtraFields(blockHeader)
 	if err != nil {
 		log.Error("[VerifyBlockInfo] Fail to decode extra field", "BlockInfoHash", blockInfo.Hash.Hex(), "BlockInfoNum", blockInfo.Number, "BlockInfoRound", blockInfo.Round, "blockHeaderNum", blockHeader.Number)
 		return err
 	}
-	if decodedExtraField.Round != blockInfo.Round {
+	if round != blockInfo.Round {
-		log.Warn("[VerifyBlockInfo] Block extra round mismatch with blockInfo", "BlockInfoHash", blockInfo.Hash.Hex(), "BlockInfoNum", blockInfo.Number, "BlockInfoRound", blockInfo.Round, "blockHeaderNum", blockHeader.Number, "blockRound", decodedExtraField.Round)
+		log.Warn("[VerifyBlockInfo] Block extra round mismatch with blockInfo", "BlockInfoHash", blockInfo.Hash.Hex(), "BlockInfoNum", blockInfo.Number, "BlockInfoRound", blockInfo.Round, "blockHeaderNum", blockHeader.Number, "blockRound", round)
-		return fmt.Errorf("[VerifyBlockInfo] chain block's round does not match from blockInfo at hash: %v and block round: %v, blockInfo Round: %v", blockInfo.Hash.Hex(), decodedExtraField.Round, blockInfo.Round)
+		return fmt.Errorf("[VerifyBlockInfo] chain block's round does not match from blockInfo at hash: %v and block round: %v, blockInfo Round: %v", blockInfo.Hash.Hex(), round, blockInfo.Round)
 	}
 
 	return nil
@@ -1046,6 +1063,12 @@ func (x *XDPoS_v2) verifyQC(blockChainReader consensus.ChainReader, quorumCert *
 		return utils.ErrInvalidQC
 	}
 
+	epochInfo, err = x.getEpochSwitchInfo(blockChainReader, nil, quorumCert.ProposedBlockInfo.Hash)
+	if err != nil {
+		log.Error("[verifyQC] Error when getting epoch switch Info to verify QC", "Error", err)
+		return fmt.Errorf("Fail to verify QC due to failure in getting epoch switch info")
+	}
+
 	var wg sync.WaitGroup
 	wg.Add(len(signatures))
 	var haveError error
@@ -1148,16 +1171,15 @@ func (x *XDPoS_v2) processQC(blockChainReader consensus.ChainReader, quorumCert
 	}
 	if proposedBlockHeader.Number.Cmp(x.config.V2.SwitchBlock) > 0 {
 		// Extra field contain parent information
-		var decodedExtraField utils.ExtraFields_v2
+		quorumCert, round, _, err := x.getExtraFields(proposedBlockHeader)
-		err := utils.DecodeBytesExtraFields(proposedBlockHeader.Extra, &decodedExtraField)
 		if err != nil {
 			return err
 		}
-		if x.lockQuorumCert == nil || decodedExtraField.QuorumCert.ProposedBlockInfo.Round > x.lockQuorumCert.ProposedBlockInfo.Round {
+		if x.lockQuorumCert == nil || quorumCert.ProposedBlockInfo.Round > x.lockQuorumCert.ProposedBlockInfo.Round {
-			x.lockQuorumCert = decodedExtraField.QuorumCert
+			x.lockQuorumCert = quorumCert
 		}
 
-		proposedBlockRound := &decodedExtraField.Round
+		proposedBlockRound := &round
 		// 3. Update commit block info
 		_, err = x.commitBlocks(blockChainReader, proposedBlockHeader, proposedBlockRound)
 		if err != nil {
@@ -1410,34 +1432,33 @@ func (x *XDPoS_v2) commitBlocks(blockChainReader consensus.ChainReader, proposed
 	// Find the last two parent block and check their rounds are the continuous
 	parentBlock := blockChainReader.GetHeaderByHash(proposedBlockHeader.ParentHash)
 
-	var decodedExtraField utils.ExtraFields_v2
+	_, round, _, err := x.getExtraFields(parentBlock)
-	err := utils.DecodeBytesExtraFields(parentBlock.Extra, &decodedExtraField)
 	if err != nil {
 		log.Error("Fail to execute first DecodeBytesExtraFields for commiting block", "ProposedBlockHash", proposedBlockHeader.Hash())
 		return false, err
 	}
-	if *proposedBlockRound-1 != decodedExtraField.Round {
+	if *proposedBlockRound-1 != round {
-		log.Debug("[commitBlocks] Rounds not continuous(parent) found when committing block", "proposedBlockRound", proposedBlockRound, "decodedExtraField.Round", decodedExtraField.Round, "proposedBlockHeaderHash", proposedBlockHeader.Hash())
+		log.Debug("[commitBlocks] Rounds not continuous(parent) found when committing block", "proposedBlockRound", proposedBlockRound, "decodedExtraField.Round", round, "proposedBlockHeaderHash", proposedBlockHeader.Hash())
 		return false, nil
 	}
 
 	// If parent round is continuous, we check grandparent
 	grandParentBlock := blockChainReader.GetHeaderByHash(parentBlock.ParentHash)
-	err = utils.DecodeBytesExtraFields(grandParentBlock.Extra, &decodedExtraField)
+	_, round, _, err = x.getExtraFields(grandParentBlock)
 	if err != nil {
 		log.Error("Fail to execute second DecodeBytesExtraFields for commiting block", "parentBlockHash", parentBlock.Hash())
 		return false, err
 	}
-	if *proposedBlockRound-2 != decodedExtraField.Round {
+	if *proposedBlockRound-2 != round {
-		log.Debug("[commitBlocks] Rounds not continuous(grand parent) found when committing block", "proposedBlockRound", proposedBlockRound, "decodedExtraField.Round", decodedExtraField.Round, "proposedBlockHeaderHash", proposedBlockHeader.Hash())
+		log.Debug("[commitBlocks] Rounds not continuous(grand parent) found when committing block", "proposedBlockRound", proposedBlockRound, "decodedExtraField.Round", round, "proposedBlockHeaderHash", proposedBlockHeader.Hash())
 		return false, nil
 	}
 	// Commit the grandParent block
-	if x.highestCommitBlock == nil || (x.highestCommitBlock.Round < decodedExtraField.Round && x.highestCommitBlock.Number.Cmp(grandParentBlock.Number) == -1) {
+	if x.highestCommitBlock == nil || (x.highestCommitBlock.Round < round && x.highestCommitBlock.Number.Cmp(grandParentBlock.Number) == -1) {
 		x.highestCommitBlock = &utils.BlockInfo{
 			Number: grandParentBlock.Number,
 			Hash:   grandParentBlock.Hash(),
-			Round:  decodedExtraField.Round,
+			Round:  round,
 		}
 		log.Debug("Successfully committed block", "Committed block Hash", x.highestCommitBlock.Hash, "Committed round", x.highestCommitBlock.Round)
 		return true, nil
@@ -1522,18 +1543,16 @@ func (x *XDPoS_v2) IsEpochSwitch(header *types.Header) (bool, uint64, error) {
 		return true, header.Number.Uint64() / x.config.Epoch, nil
 	}
 
-	var decodedExtraField utils.ExtraFields_v2
+	quorumCert, round, _, err := x.getExtraFields(header)
-	err := utils.DecodeBytesExtraFields(header.Extra, &decodedExtraField)
 	if err != nil {
 		log.Error("[IsEpochSwitch] decode header error", "err", err, "header", header, "extra", common.Bytes2Hex(header.Extra))
 		return false, 0, err
 	}
-	parentRound := decodedExtraField.QuorumCert.ProposedBlockInfo.Round
+	parentRound := quorumCert.ProposedBlockInfo.Round
-	round := decodedExtraField.Round
 	epochStartRound := round - round%utils.Round(x.config.Epoch)
 	epochNum := x.config.V2.SwitchBlock.Uint64()/x.config.Epoch + uint64(round)/x.config.Epoch
 	// if parent is last v1 block and this is first v2 block, this is treated as epoch switch
-	if decodedExtraField.QuorumCert.ProposedBlockInfo.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
+	if quorumCert.ProposedBlockInfo.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
 		log.Info("[IsEpochSwitch] true, parent equals V2.SwitchBlock", "round", round, "number", header.Number.Uint64(), "hash", header.Hash())
 		return true, epochNum, nil
 	}
@@ -1548,13 +1567,13 @@ func (x *XDPoS_v2) IsEpochSwitchAtRound(round utils.Round, parentHeader *types.H
 	if parentHeader.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
 		return true, epochNum, nil
 	}
-	var decodedExtraField utils.ExtraFields_v2
+
-	err := utils.DecodeBytesExtraFields(parentHeader.Extra, &decodedExtraField)
+	_, round, _, err := x.getExtraFields(parentHeader)
 	if err != nil {
 		log.Error("[IsEpochSwitch] decode header error", "err", err, "header", parentHeader, "extra", common.Bytes2Hex(parentHeader.Extra))
 		return false, 0, err
 	}
-	parentRound := decodedExtraField.Round
+	parentRound := round
 	epochStartRound := round - round%utils.Round(x.config.Epoch)
 	return parentRound < epochStartRound, epochNum, nil
 }
@@ -1572,6 +1591,10 @@ func (x *XDPoS_v2) getEpochSwitchInfo(chain consensus.ChainReader, header *types
 	if h == nil {
 		log.Debug("[getEpochSwitchInfo] header missing, get header", "hash", hash.Hex())
 		h = chain.GetHeaderByHash(hash)
+		if h == nil {
+			log.Warn("[getEpochSwitchInfo] can not find header from db", "hash", hash.Hex())
+			return nil, fmt.Errorf("[getEpochSwitchInfo] can not find header from db hash %v", hash.Hex())
+		}
 	}
 	isEpochSwitch, _, err := x.IsEpochSwitch(h)
 	if err != nil {
@@ -1579,36 +1602,20 @@ func (x *XDPoS_v2) getEpochSwitchInfo(chain consensus.ChainReader, header *types
 	}
 	if isEpochSwitch {
 		log.Debug("[getEpochSwitchInfo] header is epoch switch", "hash", hash.Hex(), "number", h.Number.Uint64())
-		var epochSwitchInfo *utils.EpochSwitchInfo
+		quorumCert, round, masternodes, err := x.getExtraFields(h)
-		// Special case, in case of last v1 block, we manually build the epoch switch info
+		if err != nil {
-		if h.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
+			return nil, err
-			masternodes := decodeMasternodesFromHeaderExtra(h)
+		}
-			epochSwitchInfo = &utils.EpochSwitchInfo{
+		epochSwitchInfo := &utils.EpochSwitchInfo{
-				Masternodes: masternodes,
+			Masternodes: masternodes,
-				EpochSwitchBlockInfo: &utils.BlockInfo{
+			EpochSwitchBlockInfo: &utils.BlockInfo{
-					Hash:   hash,
+				Hash:   hash,
-					Number: h.Number,
+				Number: h.Number,
-					Round:  utils.Round(0),
+				Round:  round,
-				},
+			},
-				EpochSwitchParentBlockInfo: nil,
+		}
-			}
+		if quorumCert != nil {
-		} else { // v2 normal flow
+			epochSwitchInfo.EpochSwitchParentBlockInfo = quorumCert.ProposedBlockInfo
-			masternodes := x.GetMasternodesFromEpochSwitchHeader(h)
-			// create the epoch switch info and cache it
-			var decodedExtraField utils.ExtraFields_v2
-			err = utils.DecodeBytesExtraFields(h.Extra, &decodedExtraField)
-			if err != nil {
-				return nil, err
-			}
-			epochSwitchInfo = &utils.EpochSwitchInfo{
-				Masternodes: masternodes,
-				EpochSwitchBlockInfo: &utils.BlockInfo{
-					Hash:   hash,
-					Number: h.Number,
-					Round:  decodedExtraField.Round,
-				},
-				EpochSwitchParentBlockInfo: decodedExtraField.QuorumCert.ProposedBlockInfo,
-			}
 		}
 
 		x.epochSwitches.Add(hash, epochSwitchInfo)
@@ -1712,6 +1719,26 @@ func (x *XDPoS_v2) FindParentBlockToAssign(chain consensus.ChainReader) *types.B
 	return parent
 }
 
+func (x *XDPoS_v2) getExtraFields(header *types.Header) (*utils.QuorumCert, utils.Round, []common.Address, error) {
+
+	var masternodes []common.Address
+
+	// last v1 block
+	if header.Number.Cmp(x.config.V2.SwitchBlock) == 0 {
+		masternodes = decodeMasternodesFromHeaderExtra(header)
+		return nil, utils.Round(0), masternodes, nil
+	}
+
+	// v2 block
+	masternodes = x.GetMasternodesFromEpochSwitchHeader(header)
+	var decodedExtraField utils.ExtraFields_v2
+	err := utils.DecodeBytesExtraFields(header.Extra, &decodedExtraField)
+	if err != nil {
+		return nil, utils.Round(0), masternodes, err
+	}
+	return decodedExtraField.QuorumCert, decodedExtraField.Round, masternodes, nil
+}
+
 func (x *XDPoS_v2) allowedToSend(chain consensus.ChainReader, blockHeader *types.Header, sendType string) error {
 	allowedToSend := false
 	// Don't hold the signFn for the whole signing operation

consensus/XDPoS/engines/engine_v2/snapshot.go

@@ -4,16 +4,13 @@ import (
 	"encoding/json"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
-	"github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/utils"
 	"github.com/XinFinOrg/XDPoSChain/ethdb"
-	lru "github.com/hashicorp/golang-lru"
 )
 
 // Snapshot is the state of the smart contract validator list
 // The validator list is used on next epoch master nodes
 // If we don't have the snapshot, then we have to trace back the gap block smart contract state which is very costly
 type SnapshotV2 struct {
-	Round  utils.Round `json:"round"`  // Round number
 	Number uint64      `json:"number"` // Block number where the snapshot was created
 	Hash   common.Hash `json:"hash"`   // Block hash where the snapshot was created
 
@@ -22,9 +19,8 @@ type SnapshotV2 struct {
 }
 
 // create new snapshot for next epoch to use
-func newSnapshot(number uint64, hash common.Hash, round utils.Round, qc *utils.QuorumCert, masternodes []common.Address) *SnapshotV2 {
+func newSnapshot(number uint64, hash common.Hash, masternodes []common.Address) *SnapshotV2 {
 	snap := &SnapshotV2{
-		Round:                round,
 		Number:               number,
 		Hash:                 hash,
 		NextEpochMasterNodes: masternodes,
@@ -33,8 +29,8 @@ func newSnapshot(number uint64, hash common.Hash, round utils.Round, qc *utils.Q
 }
 
 // loadSnapshot loads an existing snapshot from the database.
-func loadSnapshot(sigcache *lru.ARCCache, db ethdb.Database, hash common.Hash) (*SnapshotV2, error) {
+func loadSnapshot(db ethdb.Database, hash common.Hash) (*SnapshotV2, error) {
-	blob, err := db.Get(append([]byte("XDPoS-"), hash[:]...))
+	blob, err := db.Get(append([]byte("XDPoS-V2-"), hash[:]...))
 	if err != nil {
 		return nil, err
 	}
@@ -52,7 +48,7 @@ func storeSnapshot(s *SnapshotV2, db ethdb.Database) error {
 	if err != nil {
 		return err
 	}
-	return db.Put(append([]byte("XDPoS-"), s.Hash[:]...), blob)
+	return db.Put(append([]byte("XDPoS-V2-"), s.Hash[:]...), blob)
 }
 
 // retrieves master nodes list in map type

consensus/XDPoS/engines/engine_v2/snapshot_test.go

@@ -6,14 +6,13 @@ import (
 	"testing"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
-	"github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/utils"
 	"github.com/XinFinOrg/XDPoSChain/core/rawdb"
 	"github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"
 )
 
 func TestGetMasterNodes(t *testing.T) {
 	masterNodes := []common.Address{{0x4}, {0x3}, {0x2}, {0x1}}
-	snap := newSnapshot(1, common.Hash{}, utils.Round(1), nil, masterNodes)
+	snap := newSnapshot(1, common.Hash{}, masterNodes)
 
 	for _, address := range masterNodes {
 		if _, ok := snap.GetMappedMasterNodes()[address]; !ok {
@@ -24,7 +23,7 @@ func TestGetMasterNodes(t *testing.T) {
 }
 
 func TestStoreLoadSnapshot(t *testing.T) {
-	snap := newSnapshot(1, common.Hash{0x1}, utils.Round(1), nil, nil)
+	snap := newSnapshot(1, common.Hash{0x1}, nil)
 	dir, err := ioutil.TempDir("", "snapshot-test")
 	if err != nil {
 		panic(fmt.Sprintf("can't create temporary directory: %v", err))
@@ -40,7 +39,7 @@ func TestStoreLoadSnapshot(t *testing.T) {
 		t.Error("store snapshot failed", err)
 	}
 
-	restoredSnapshot, err := loadSnapshot(nil, lddb, snap.Hash)
+	restoredSnapshot, err := loadSnapshot(lddb, snap.Hash)
 	if err != nil || restoredSnapshot.Hash != snap.Hash {
 		t.Error("load snapshot failed", err)
 	}

consensus/tests/adaptor_test.go

@@ -236,7 +236,7 @@ func TestGetParentBlock(t *testing.T) {
 	assert.Equal(t, block, block900)
 
 	// Initialise
-	err := adaptor.EngineV2.Initial(blockchain, []common.Address{})
+	err := adaptor.EngineV2.Initial(blockchain, block.Header())
 	assert.Nil(t, err)
 
 	// V2

consensus/tests/initial_test.go

@@ -0,0 +1,123 @@
+package tests
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/XinFinOrg/XDPoSChain/common"
+	"github.com/XinFinOrg/XDPoSChain/consensus/XDPoS"
+	"github.com/XinFinOrg/XDPoSChain/consensus/XDPoS/utils"
+	"github.com/XinFinOrg/XDPoSChain/core/types"
+	"github.com/XinFinOrg/XDPoSChain/params"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInitialFirstV2Blcok(t *testing.T) {
+	blockchain, _, currentBlock, _, _, _ := PrepareXDCTestBlockChainForV2Engine(t, 900, params.TestXDPoSMockChainConfig, 0)
+	adaptor := blockchain.Engine().(*XDPoS.XDPoS)
+	header := currentBlock.Header()
+
+	// snapshot should not be created before initial
+	snap, _ := adaptor.EngineV2.GetSnapshot(blockchain, currentBlock.Header())
+	assert.Nil(t, snap)
+
+	err := adaptor.EngineV2.Initial(blockchain, header)
+	assert.Nil(t, err)
+
+	round, _, highQC, _, _ := adaptor.EngineV2.GetProperties()
+	blockInfo := &utils.BlockInfo{
+		Hash:   header.Hash(),
+		Round:  utils.Round(0),
+		Number: header.Number,
+	}
+	expectedQuorumCert := &utils.QuorumCert{
+		ProposedBlockInfo: blockInfo,
+		Signatures:        nil,
+	}
+	assert.Equal(t, utils.Round(1), round)
+	assert.Equal(t, expectedQuorumCert, highQC)
+
+	// Test snapshot
+	snap, err = adaptor.EngineV2.GetSnapshot(blockchain, currentBlock.Header())
+	assert.Nil(t, err)
+	assert.Equal(t, uint64(450), snap.Number)
+
+	// Test Running channels
+	WaitPeriod := <-adaptor.WaitPeriodCh
+	assert.Equal(t, params.TestXDPoSMockChainConfig.XDPoS.V2.WaitPeriod, WaitPeriod)
+
+	t.Logf("Waiting %d secs for timeout to happen", params.TestXDPoSMockChainConfig.XDPoS.V2.TimeoutPeriod)
+	timeoutMsg := <-adaptor.EngineV2.BroadcastCh
+	assert.NotNil(t, timeoutMsg)
+	assert.Equal(t, utils.Round(1), timeoutMsg.(*utils.Timeout).Round)
+}
+
+func TestInitialOtherV2Block(t *testing.T) {
+	// insert new block with new extra fields
+	blockchain, _, currentBlock, signer, signFn, _ := PrepareXDCTestBlockChainForV2Engine(t, 900, params.TestXDPoSMockChainConfig, 0)
+	adaptor := blockchain.Engine().(*XDPoS.XDPoS)
+
+	blockCoinBase := "0x111000000000000000000000000000000123"
+	for blockNum := 901; blockNum <= 910; blockNum++ {
+		currentBlock = CreateBlock(blockchain, params.TestXDPoSMockChainConfig, currentBlock, blockNum, int64(blockNum-900), blockCoinBase, signer, signFn, nil)
+		err := blockchain.InsertBlock(currentBlock)
+		assert.Nil(t, err)
+	}
+
+	// v2
+	blockInfo := &utils.BlockInfo{
+		Hash:   currentBlock.Header().Hash(),
+		Round:  utils.Round(10),
+		Number: big.NewInt(910),
+	}
+	quorumCert := &utils.QuorumCert{
+		ProposedBlockInfo: blockInfo,
+		Signatures:        nil, // after decode it got default value []utils.Signature{}
+	}
+	extra := utils.ExtraFields_v2{
+		Round:      11,
+		QuorumCert: quorumCert,
+	}
+	extraBytes, err := extra.EncodeToBytes()
+	assert.Nil(t, err)
+
+	header := &types.Header{
+		Root:       common.HexToHash("ea465415b60d88429f181fec9fae67c0f19cbf5a4fa10971d96d4faa57d96ffa"),
+		Number:     big.NewInt(int64(911)),
+		ParentHash: currentBlock.Hash(),
+		Coinbase:   common.HexToAddress("0x111000000000000000000000000000000123"),
+	}
+	header.Extra = extraBytes
+
+	block, err := createBlockFromHeader(blockchain, header, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	blockchain.InsertBlock(block)
+	// Initialise
+	err = adaptor.EngineV2.Initial(blockchain, block.Header())
+	assert.Nil(t, err)
+
+	round, _, highQC, _, _ := adaptor.EngineV2.GetProperties()
+	expectedQuorumCert := &utils.QuorumCert{
+		ProposedBlockInfo: blockInfo,
+		Signatures:        []utils.Signature{},
+	}
+	assert.Equal(t, utils.Round(11), round)
+	assert.Equal(t, expectedQuorumCert, highQC)
+
+	// Test snapshot
+	snap, err := adaptor.EngineV2.GetSnapshot(blockchain, block.Header())
+	assert.Nil(t, err)
+	assert.Equal(t, uint64(450), snap.Number)
+}
+
+func TestSnapshotShouldAlreadyCreatedByUpdateM1(t *testing.T) {
+	// insert new block with new extra fields
+	blockchain, _, currentBlock, _, _, _ := PrepareXDCTestBlockChainForV2Engine(t, 1800, params.TestXDPoSMockChainConfig, 0)
+	adaptor := blockchain.Engine().(*XDPoS.XDPoS)
+
+	snap, err := adaptor.EngineV2.GetSnapshot(blockchain, currentBlock.Header())
+	assert.Nil(t, err)
+	assert.Equal(t, uint64(1350), snap.Number)
+}

consensus/tests/mine_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestYourTurnInitialV2(t *testing.T) {
 	config := params.TestXDPoSMockChainConfig
-	blockchain, _, parentBlock, _ := PrepareXDCTestBlockChain(t, int(config.XDPoS.Epoch)-1, config)
+	blockchain, _, parentBlock, _, _, _ := PrepareXDCTestBlockChainForV2Engine(t, int(config.XDPoS.Epoch)-1, config, 0)
 	minePeriod := config.XDPoS.V2.MinePeriod
 	adaptor := blockchain.Engine().(*XDPoS.XDPoS)
 

consensus/tests/test_helper.go

@@ -418,8 +418,7 @@ func PrepareXDCTestBlockChainForV2Engine(t *testing.T, numOfBlocks int, chainCon
 			lastv1BlockNumber := block.Header().Number.Uint64() - 1
 			checkpointBlockNumber := lastv1BlockNumber - lastv1BlockNumber%chainConfig.XDPoS.Epoch
 			checkpointHeader := blockchain.GetHeaderByNumber(checkpointBlockNumber)
-			masternodes := engine.EngineV1.GetMasternodesFromCheckpointHeader(checkpointHeader)
+			err := engine.EngineV2.Initial(blockchain, checkpointHeader)
-			err := engine.EngineV2.Initial(blockchain, masternodes)
 			if err != nil {
 				panic(err)
 			}