Merge pull request #397 from thanhson1085/org_master

refactor cache blocksigners

common/types.go

@@ -39,6 +39,7 @@ const (
 	UnvoteMethod        = "0x02aa9be2"
 	ProposeMethod       = "0x01267951"
 	ResignMethod        = "0xae6e43f5"
+	SignMethod          = "0xe341eaa4"
 )
 
 var (

consensus/posv/posv.go

@@ -50,7 +50,6 @@ import (
 const (
 	inmemorySnapshots      = 128 // Number of recent vote snapshots to keep in memory
 	blockSignersCacheLimit = 36000
-	votingCacheLimit       = 1500000
 	M2ByteLength           = 4
 )
 
@@ -226,9 +225,7 @@ type Posv struct {
 	signFn clique.SignerFn // Signer function to authorize hashes with
 	lock   sync.RWMutex    // Protects the signer fields
 
-	EnableCache   bool
 	BlockSigners  *lru.Cache
-	Votes         *lru.Cache
 	HookReward    func(chain consensus.ChainReader, state *state.StateDB, header *types.Header) (error, map[string]interface{})
 	HookPenalty   func(chain consensus.ChainReader, blockNumberEpoc uint64) ([]common.Address, error)
 	HookValidator func(header *types.Header, signers []common.Address) ([]byte, error)
@@ -245,7 +242,6 @@ func New(config *params.PosvConfig, db ethdb.Database) *Posv {
 	}
 	// Allocate the snapshot caches and create the engine
 	BlockSigners, _ := lru.New(blockSignersCacheLimit)
-	Votes, _ := lru.New(votingCacheLimit)
 	recents, _ := lru.NewARC(inmemorySnapshots)
 	signatures, _ := lru.NewARC(inmemorySnapshots)
 	validatorSignatures, _ := lru.NewARC(inmemorySnapshots)
@@ -253,9 +249,7 @@ func New(config *params.PosvConfig, db ethdb.Database) *Posv {
 	return &Posv{
 		config:              &conf,
 		db:                  db,
-		EnableCache:         false,
 		BlockSigners:        BlockSigners,
-		Votes:               Votes,
 		recents:             recents,
 		signatures:          signatures,
 		verifiedHeaders:     verifiedHeaders,
@@ -859,12 +853,9 @@ func (c *Posv) Finalize(chain consensus.ChainReader, header *types.Header, state
 	number := header.Number.Uint64()
 	rCheckpoint := chain.Config().Posv.RewardCheckpoint
 
-	if c.HookReward != nil && number%rCheckpoint == 0 {
+	// _ = c.CacheData(header, txs, receipts)
-		if !c.EnableCache && int(c.BlockSigners.Len()) >= int(rCheckpoint*3) {
-			log.Debug("EnableCache true c.BlockSigners.Len() ", "BlockSigners.Len", c.BlockSigners.Len())
-			c.EnableCache = true
-		}
 
+	if c.HookReward != nil && number%rCheckpoint == 0 {
 		err, rewards := c.HookReward(chain, state, header)
 		if err != nil {
 			return nil, err
@@ -880,8 +871,6 @@ func (c *Posv) Finalize(chain consensus.ChainReader, header *types.Header, state
 		}
 	}
 
-	_ = c.cacheData(txs, receipts)
-
 	// the state remains as is and uncles are dropped
 	header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
 	header.UncleHash = types.CalcUncleHash(nil)
@@ -1036,12 +1025,10 @@ func (c *Posv) GetMasternodesFromCheckpointHeader(preCheckpointHeader *types.Hea
 	return masternodes
 }
 
-func (c *Posv) cacheData(txs []*types.Transaction, receipts []*types.Receipt) error {
+func (c *Posv) CacheData(header *types.Header, txs []*types.Transaction, receipts []*types.Receipt) error {
+	var signTxs []*types.Transaction
 	for _, tx := range txs {
 		if tx.IsSigningTransaction() {
-			blkHash := common.BytesToHash(tx.Data()[len(tx.Data())-32:])
-			from := *tx.From()
-
 			var b uint
 			for _, r := range receipts {
 				if r.TxHash == tx.Hash() {
@@ -1054,31 +1041,20 @@ func (c *Posv) cacheData(txs []*types.Transaction, receipts []*types.Receipt) er
 				continue
 			}
 
-			var lAddr []common.Address
+			signTxs = append(signTxs, tx)
-			if cached, ok := c.BlockSigners.Get(blkHash); ok {
+		}
-				lAddr = cached.([]common.Address)
-				lAddr = append(lAddr, from)
-			} else {
-				lAddr = []common.Address{from}
 	}
-			c.BlockSigners.Add(blkHash, lAddr)
-		} else {
 
-			b, addr := tx.IsVotingTransaction()
+	log.Debug("Save tx signers to cache", "hash", header.Hash().String(), "number", header.Number, "len(txs)", len(signTxs))
-			if b && addr != nil {
+	c.BlockSigners.Add(header.Hash(), signTxs)
-				var vote common.Vote
-				vote.Masternode = *addr
-				vote.Voter = *tx.From()
-
-				log.Debug("Remove from Votes cache ", "Masternode", vote.Masternode.String(), "Voter", vote.Voter.String())
-				c.Votes.Remove(vote)
-			}
-		}
-	}
 
 	return nil
 }
 
+func (c *Posv) GetDb() ethdb.Database {
+	return c.db
+}
+
 // Extract validators from byte array.
 func RemovePenaltiesFromBlock(chain consensus.ChainReader, masternodes []common.Address, epochNumber uint64) []common.Address {
 	if epochNumber <= 0 {

contracts/utils.go

@@ -199,13 +199,12 @@ func BuildTxOpeningRandomize(nonce uint64, randomizeAddr common.Address, randomi
 }
 
 // Get signers signed for blockNumber from blockSigner contract.
-func GetSignersFromContract(c *posv.Posv, addrBlockSigner common.Address, client bind.ContractBackend, blockHash common.Hash) ([]common.Address, error) {
+func GetSignersFromContract(addrBlockSigner common.Address, client bind.ContractBackend, blockHash common.Hash) ([]common.Address, error) {
 	blockSigner, err := contract.NewBlockSigner(addrBlockSigner, client)
 	if err != nil {
 		log.Error("Fail get instance of blockSigner", "error", err)
 		return nil, err
 	}
-	if caddrs, ok := c.BlockSigners.Get(blockHash); !ok || !c.EnableCache {
 	opts := new(bind.CallOpts)
 	addrs, err := blockSigner.GetSigners(opts, blockHash)
 	if err != nil {
@@ -213,9 +212,6 @@ func GetSignersFromContract(c *posv.Posv, addrBlockSigner common.Address, client
 		return nil, err
 	}
 	return addrs, nil
-	} else {
-		return caddrs.([]common.Address), nil
-	}
 }
 
 // Get random from randomize contract.
@@ -310,7 +306,7 @@ func DecryptRandomizeFromSecretsAndOpening(secrets [][32]byte, opening [32]byte)
 }
 
 // Calculate reward for reward checkpoint.
-func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, blockSignerAddr common.Address, number uint64, rCheckpoint uint64, client bind.ContractBackend, totalSigner *uint64) (map[common.Address]*rewardLog, error) {
+func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, number uint64, rCheckpoint uint64, totalSigner *uint64) (map[common.Address]*rewardLog, error) {
 	// Not reward for singer of genesis block and only calculate reward at checkpoint block.
 	prevCheckpoint := number - (rCheckpoint * 2)
 	startBlockNumber := prevCheckpoint + 1
@@ -321,14 +317,52 @@ func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, blockSign
 
 	if len(masternodes) > 0 {
 
-		if !c.EnableCache {
+		data := make(map[common.Hash][]common.Address)
+		for i := startBlockNumber; i <= prevCheckpoint+(rCheckpoint*2)-1; i++ {
+			header := chain.GetHeaderByNumber(i)
+
+			if signData, ok := c.BlockSigners.Get(header.Hash()); ok {
+				txs := signData.([]*types.Transaction)
+				for _, tx := range txs {
+					blkHash := common.BytesToHash(tx.Data()[len(tx.Data())-32:])
+					from := *tx.From()
+					data[blkHash] = append(data[blkHash], from)
+				}
+			} else {
+				log.Debug("Failed get from cached", "hash", header.Hash().String(), "number", i)
+				block := chain.GetBlock(header.Hash(), i)
+				txs := block.Transactions()
+				receipts := core.GetBlockReceipts(c.GetDb(), header.Hash(), i)
+
+				var signTxs []*types.Transaction
+				for _, tx := range txs {
+					if tx.IsSigningTransaction() {
+						var b uint
+						for _, r := range receipts {
+							if r.TxHash == tx.Hash() {
+								b = r.Status
+								break
+							}
+						}
+
+						if b == types.ReceiptStatusFailed {
+							continue
+						}
+
+						signTxs = append(signTxs, tx)
+						blkHash := common.BytesToHash(tx.Data()[len(tx.Data())-32:])
+						from := *tx.From()
+						data[blkHash] = append(data[blkHash], from)
+					}
+				}
+				c.BlockSigners.Add(header.Hash(), signTxs)
+
+			}
+		}
+
 		for i := startBlockNumber; i <= endBlockNumber; i++ {
 			block := chain.GetHeaderByNumber(i)
-				addrs, err := GetSignersFromContract(c, blockSignerAddr, client, block.Hash())
+			addrs := data[block.Hash()]
-				if err != nil {
-					log.Error("Fail to get signers from smartcontract.", "error", err, "blockNumber", i)
-					return nil, err
-				}
 			// Filter duplicate address.
 			if len(addrs) > 0 {
 				addrSigners := make(map[common.Address]bool)
@@ -354,56 +388,6 @@ func GetRewardForCheckpoint(c *posv.Posv, chain consensus.ChainReader, blockSign
 				}
 			}
 		}
-		} else {
-			var wg sync.WaitGroup
-			squeue := make(chan []common.Address, 1)
-			wg.Add(int(rCheckpoint))
-
-			for i := startBlockNumber; i <= endBlockNumber; i++ {
-				go func(i uint64) {
-					block := chain.GetHeaderByNumber(i)
-					addrs, err := GetSignersFromContract(c, blockSignerAddr, client, block.Hash())
-					if err != nil {
-						log.Crit("Fail to get signers from smartcontract.", "error", err, "blockNumber", i)
-					}
-					squeue <- addrs
-				}(i)
-			}
-
-			fsigner := func() {
-				for addrs := range squeue {
-					// Filter duplicate address.
-					if len(addrs) > 0 {
-						addrSigners := make(map[common.Address]bool)
-						for _, masternode := range masternodes {
-							for _, addr := range addrs {
-								if addr == masternode {
-									if _, ok := addrSigners[addr]; !ok {
-										addrSigners[addr] = true
-									}
-									break
-								}
-							}
-						}
-
-						for addr := range addrSigners {
-							_, exist := signers[addr]
-							if exist {
-								signers[addr].Sign++
-							} else {
-								signers[addr] = &rewardLog{1, new(big.Int)}
-							}
-							*totalSigner++
-						}
-					}
-					wg.Done()
-				}
-			}
-
-			go fsigner()
-
-			wg.Wait()
-		}
 	}
 
 	log.Info("Calculate reward at checkpoint", "startBlock", startBlockNumber, "endBlock", endBlockNumber)
@@ -449,8 +433,8 @@ func GetCandidatesOwnerBySigner(validator *contractValidator.TomoValidator, sign
 }
 
 // Calculate reward for holders.
-func CalculateRewardForHolders(c *posv.Posv, foudationWalletAddr common.Address, validator *contractValidator.TomoValidator, state *state.StateDB, signer common.Address, calcReward *big.Int) (error, map[common.Address]*big.Int) {
+func CalculateRewardForHolders(foudationWalletAddr common.Address, validator *contractValidator.TomoValidator, state *state.StateDB, signer common.Address, calcReward *big.Int) (error, map[common.Address]*big.Int) {
-	rewards, err := GetRewardBalancesRate(c, foudationWalletAddr, signer, calcReward, validator)
+	rewards, err := GetRewardBalancesRate(foudationWalletAddr, signer, calcReward, validator)
 	if err != nil {
 		return err, nil
 	}
@@ -463,7 +447,7 @@ func CalculateRewardForHolders(c *posv.Posv, foudationWalletAddr common.Address,
 }
 
 // Get reward balance rates for master node, founder and holders.
-func GetRewardBalancesRate(c *posv.Posv, foudationWalletAddr common.Address, masterAddr common.Address, totalReward *big.Int, validator *contractValidator.TomoValidator) (map[common.Address]*big.Int, error) {
+func GetRewardBalancesRate(foudationWalletAddr common.Address, masterAddr common.Address, totalReward *big.Int, validator *contractValidator.TomoValidator) (map[common.Address]*big.Int, error) {
 	owner := GetCandidatesOwnerBySigner(validator, masterAddr)
 	balances := make(map[common.Address]*big.Int)
 	rewardMaster := new(big.Int).Mul(totalReward, new(big.Int).SetInt64(common.RewardMasterPercent))
@@ -484,29 +468,12 @@ func GetRewardBalancesRate(c *posv.Posv, foudationWalletAddr common.Address, mas
 		// Get voters capacities.
 		voterCaps := make(map[common.Address]*big.Int)
 		for _, voteAddr := range voters {
-			var vote common.Vote
 			var voterCap *big.Int
 
-			vote.Masternode = masterAddr
-			vote.Voter = voteAddr
-
-			if c != nil {
-				if vCap, ok := c.Votes.Get(vote); ok {
-					voterCap = vCap.(*big.Int)
-				} else {
 			voterCap, err = validator.GetVoterCap(opts, masterAddr, voteAddr)
 			if err != nil {
 				log.Crit("Fail to get vote capacity", "error", err)
 			}
-					log.Debug("Add to Votes cache ", "vote.Masternode", vote.Masternode.String(), "vote.Voter", vote.Voter.String(), "voterCap", voterCap.String())
-					c.Votes.Add(vote, voterCap)
-				}
-			} else {
-				voterCap, err = validator.GetVoterCap(opts, masterAddr, voteAddr)
-				if err != nil {
-					log.Crit("Fail to get vote capacity", "error", err)
-				}
-			}
 
 			totalCap.Add(totalCap, voterCap)
 			voterCaps[voteAddr] = voterCap

contracts/validator/validator_test.go

@@ -144,7 +144,7 @@ func TestRewardBalance(t *testing.T) {
 
 	foundationAddr := common.HexToAddress(common.FoudationAddr)
 	totalReward := new(big.Int).SetInt64(15 * 1000)
-	rewards, err := contracts.GetRewardBalancesRate(nil, foundationAddr, acc3Addr, totalReward, baseValidator)
+	rewards, err := contracts.GetRewardBalancesRate(foundationAddr, acc3Addr, totalReward, baseValidator)
 	if err != nil {
 		t.Error("Fail to get reward balances rate.", err)
 	}

core/blockchain.go

@@ -506,6 +506,10 @@ func (bc *BlockChain) insert(block *types.Block) {
 	}
 	bc.currentBlock.Store(block)
 
+	// save cache BlockSigners
+	engine := bc.Engine().(*posv.Posv)
+	engine.CacheData(block.Header(), block.Transactions(), bc.GetReceiptsByHash(block.Hash()))
+
 	// If the block is better than our head or is on a different chain, force update heads
 	if updateHeads {
 		bc.hc.SetCurrentHeader(block.Header())

core/types/transaction.go

@@ -291,7 +291,22 @@ func (tx *Transaction) IsSigningTransaction() bool {
 	if tx.To() == nil {
 		return false
 	}
-	return tx.To().String() == common.BlockSigners
+
+	if tx.To().String() != common.BlockSigners {
+		return false
+	}
+
+	method := common.ToHex(tx.Data()[0:4])
+
+	if method != common.SignMethod {
+		return false
+	}
+
+	if len(tx.Data()) != (32*2 + 4) {
+		return false
+	}
+
+	return true
 }
 
 func (tx *Transaction) IsVotingTransaction() (bool, *common.Address) {

eth/backend.go

@@ -259,7 +259,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 					for i := prevEpoc; i < blockNumberEpoc; i++ {
 						blockHeader := chain.GetHeaderByNumber(i)
 						if len(penSigners) > 0 {
-							signedMasternodes, err := contracts.GetSignersFromContract(c, blockSignerAddr, client, blockHeader.Hash())
+							signedMasternodes, err := contracts.GetSignersFromContract(blockSignerAddr, client, blockHeader.Hash())
 							if err != nil {
 								return nil, err
 							}
@@ -300,14 +300,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 			rewards := make(map[string]interface{})
 			if number > 0 && number-rCheckpoint > 0 && foudationWalletAddr != (common.Address{}) {
 				start := time.Now()
-				// Get signers in blockSigner smartcontract.
-				addr := common.HexToAddress(common.BlockSigners)
 				// Get reward inflation.
 				chainReward := new(big.Int).Mul(new(big.Int).SetUint64(chain.Config().Posv.Reward), new(big.Int).SetUint64(params.Ether))
 				chainReward = rewardInflation(chainReward, number, common.BlocksPerYear)
 
 				totalSigner := new(uint64)
-				signers, err := contracts.GetRewardForCheckpoint(c, chain, addr, number, rCheckpoint, client, totalSigner)
+				signers, err := contracts.GetRewardForCheckpoint(c, chain, number, rCheckpoint, totalSigner)
 				log.Debug("Time Get Signers", "block", header.Number.Uint64(), "time", common.PrettyDuration(time.Since(start)))
 				if err != nil {
 					log.Crit("Fail to get signers for reward checkpoint", "error", err)
@@ -326,7 +324,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 				voterResults := make(map[common.Address]interface{})
 				if len(signers) > 0 {
 					for signer, calcReward := range rewardSigners {
-						err, rewards := contracts.CalculateRewardForHolders(c, foudationWalletAddr, validator, state, signer, calcReward)
+						err, rewards := contracts.CalculateRewardForHolders(foudationWalletAddr, validator, state, signer, calcReward)
 						if err != nil {
 							log.Crit("Fail to calculate reward for holders.", "error", err)
 						}

internal/ethapi/api.go

@@ -867,7 +867,7 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx
 	if b.Number().Int64() > 0 {
 		engine := s.b.GetEngine()
 		addrBlockSigner := common.HexToAddress(common.BlockSigners)
-		signers, err = contracts.GetSignersFromContract(engine.(*posv.Posv), addrBlockSigner, client, b.Hash())
+		signers, err = contracts.GetSignersFromContract(addrBlockSigner, client, b.Hash())
 		if err != nil {
 			log.Error("Fail to get signers from block signer SC.", "error", err)
 			return nil, err