Arpit/temp bor sync (#701)

* Increase grpc message size limit in pprof

* ReadBorReceipts improvements

* use internal function

* fix tests

* fetch geth upstread for ReadBorReceiptRLP

* Only query bor receipt when the query index is equal to # tx in block body

This change reduces the frequency of calling ReadBorReceipt and
ReadBorTransaction, which are CPU and db intensive.

* Revert "fetch geth upstread for ReadBorReceiptRLP"

This reverts commit 2e838a6b1313d26674f3a8df4b044e35dcbf35a0.

* Restore ReadBorReceiptRLP

* fix bor receipts

* remove unused

* fix lints

---------

Co-authored-by: Jerry <jerrycgh@gmail.com>
Co-authored-by: Manav Darji <manavdarji.india@gmail.com>
Co-authored-by: Evgeny Danienko <6655321@bk.ru>

core/blockchain.go

@@ -2059,7 +2059,7 @@ func (bc *BlockChain) collectLogs(hash common.Hash, removed bool) []*types.Log {
 	receipts := rawdb.ReadReceipts(bc.db, hash, *number, bc.chainConfig)
 
 	// Append bor receipt
-	borReceipt := rawdb.ReadBorReceipt(bc.db, hash, *number)
+	borReceipt := rawdb.ReadBorReceipt(bc.db, hash, *number, bc.chainConfig)
 	if borReceipt != nil {
 		receipts = append(receipts, borReceipt)
 	}

core/bor_blockchain.go

@@ -19,7 +19,7 @@ func (bc *BlockChain) GetBorReceiptByHash(hash common.Hash) *types.Receipt {
 	}
 
 	// read bor reciept by hash and number
-	receipt := rawdb.ReadBorReceipt(bc.db, hash, *number)
+	receipt := rawdb.ReadBorReceipt(bc.db, hash, *number, bc.chainConfig)
 	if receipt == nil {
 		return nil
 	}

core/rawdb/bor_receipt.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
@@ -33,49 +34,28 @@ func borTxLookupKey(hash common.Hash) []byte {
 	return append(borTxLookupPrefix, hash.Bytes()...)
 }
 
-// HasBorReceipt verifies the existence of all block receipt belonging
-// to a block.
-func HasBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) bool {
-	if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash {
-		return true
-	}
-
-	if has, err := db.Has(borReceiptKey(number, hash)); !has || err != nil {
-		return false
-	}
-
-	return true
-}
-
-// ReadBorReceiptRLP retrieves the block receipt belonging to a block in RLP encoding.
 func ReadBorReceiptRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
-	// First try to look up the data in ancient database. Extra hash
+	var data []byte
-	// comparison is necessary since ancient database only maintains
+
-	// the canonical data.
+	err := db.ReadAncients(func(reader ethdb.AncientReader) error {
-	data, _ := db.Ancient(freezerBorReceiptTable, number)
+		// Check if the data is in ancients
-	if len(data) > 0 {
+		if isCanon(reader, number, hash) {
-		h, _ := db.Ancient(freezerHashTable, number)
+			data, _ = reader.Ancient(freezerBorReceiptTable, number)
-		if common.BytesToHash(h) == hash {
+
-			return data
+			return nil
 		}
-	}
+
-	// Then try to look up the data in leveldb.
+		// If not, try reading from leveldb
 		data, _ = db.Get(borReceiptKey(number, hash))
-	if len(data) > 0 {
+
+		return nil
+	})
+
+	if err != nil {
+		log.Warn("during ReadBorReceiptRLP", "number", number, "hash", hash, "err", err)
+	}
+
 	return data
-	}
-	// In the background freezer is moving data from leveldb to flatten files.
-	// So during the first check for ancient db, the data is not yet in there,
-	// but when we reach into leveldb, the data was already moved. That would
-	// result in a not found error.
-	data, _ = db.Ancient(freezerBorReceiptTable, number)
-	if len(data) > 0 {
-		h, _ := db.Ancient(freezerHashTable, number)
-		if common.BytesToHash(h) == hash {
-			return data
-		}
-	}
-	return nil // Can't find the data anywhere.
 }
 
 // ReadRawBorReceipt retrieves the block receipt belonging to a block.
@@ -101,7 +81,11 @@ func ReadRawBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types.
 // ReadBorReceipt retrieves all the bor block receipts belonging to a block, including
 // its correspoinding metadata fields. If it is unable to populate these metadata
 // fields then nil is returned.
-func ReadBorReceipt(db ethdb.Reader, hash common.Hash, number uint64) *types.Receipt {
+func ReadBorReceipt(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) *types.Receipt {
+	if config != nil && config.Bor != nil && config.Bor.Sprint != nil && !config.Bor.IsSprintStart(number) {
+		return nil
+	}
+
 	// We're deriving many fields from the block body, retrieve beside the receipt
 	borReceipt := ReadRawBorReceipt(db, hash, number)
 	if borReceipt == nil {

eth/filters/test_backend.go

@@ -38,7 +38,7 @@ func (b *TestBackend) GetBorBlockReceipt(ctx context.Context, hash common.Hash)
 		return &types.Receipt{}, nil
 	}
 
-	receipt := rawdb.ReadBorReceipt(b.DB, hash, *number)
+	receipt := rawdb.ReadBorReceipt(b.DB, hash, *number, nil)
 	if receipt == nil {
 		return &types.Receipt{}, nil
 	}

eth/tracers/api.go

@@ -177,7 +177,7 @@ func (api *API) getAllBlockTransactions(ctx context.Context, block *types.Block)
 
 	stateSyncPresent := false
 
-	borReceipt := rawdb.ReadBorReceipt(api.backend.ChainDb(), block.Hash(), block.NumberU64())
+	borReceipt := rawdb.ReadBorReceipt(api.backend.ChainDb(), block.Hash(), block.NumberU64(), api.backend.ChainConfig())
 	if borReceipt != nil {
 		txHash := types.GetDerivedBorTxHash(types.BorReceiptKey(block.Number().Uint64(), block.Hash()))
 		if txHash != (common.Hash{}) {

internal/ethapi/api.go

@@ -636,7 +636,7 @@ func (s *PublicBlockChainAPI) GetTransactionReceiptsByBlock(ctx context.Context,
 
 	var txHash common.Hash
 
-	borReceipt := rawdb.ReadBorReceipt(s.b.ChainDb(), block.Hash(), block.NumberU64())
+	borReceipt := rawdb.ReadBorReceipt(s.b.ChainDb(), block.Hash(), block.NumberU64(), s.b.ChainConfig())
 	if borReceipt != nil {
 		receipts = append(receipts, borReceipt)
 		txHash = types.GetDerivedBorTxHash(types.BorReceiptKey(block.Number().Uint64(), block.Hash()))
@@ -1453,7 +1453,13 @@ func newRPCPendingTransaction(tx *types.Transaction, current *types.Header, conf
 func newRPCTransactionFromBlockIndex(b *types.Block, index uint64, config *params.ChainConfig, db ethdb.Database) *RPCTransaction {
 	txs := b.Transactions()
 
-	borReceipt := rawdb.ReadBorReceipt(db, b.Hash(), b.NumberU64())
+	if index >= uint64(len(txs)+1) {
+		return nil
+	}
+
+	// If the index out of the range of transactions defined in block body, it means that the transaction is a bor state sync transaction, and we need to fetch it from the database
+	if index == uint64(len(txs)) {
+		borReceipt := rawdb.ReadBorReceipt(db, b.Hash(), b.NumberU64(), config)
 		if borReceipt != nil {
 			tx, _, _, _ := rawdb.ReadBorTransaction(db, borReceipt.TxHash)
 
@@ -1461,7 +1467,9 @@ func newRPCTransactionFromBlockIndex(b *types.Block, index uint64, config *param
 				txs = append(txs, tx)
 			}
 		}
+	}
 
+	// If the index is still out of the range after checking bor state sync transaction, it means that the transaction index is invalid
 	if index >= uint64(len(txs)) {
 		return nil
 	}
@@ -1602,7 +1610,7 @@ func (api *PublicTransactionPoolAPI) getAllBlockTransactions(ctx context.Context
 
 	stateSyncPresent := false
 
-	borReceipt := rawdb.ReadBorReceipt(api.b.ChainDb(), block.Hash(), block.NumberU64())
+	borReceipt := rawdb.ReadBorReceipt(api.b.ChainDb(), block.Hash(), block.NumberU64(), api.b.ChainConfig())
 	if borReceipt != nil {
 		txHash := types.GetDerivedBorTxHash(types.BorReceiptKey(block.Number().Uint64(), block.Hash()))
 		if txHash != (common.Hash{}) {
@@ -1772,7 +1780,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha
 
 	if borTx {
 		// Fetch bor block receipt
-		receipt = rawdb.ReadBorReceipt(s.b.ChainDb(), blockHash, blockNumber)
+		receipt = rawdb.ReadBorReceipt(s.b.ChainDb(), blockHash, blockNumber, s.b.ChainConfig())
 	} else {
 		receipts, err := s.b.GetReceipts(ctx, blockHash)
 		if err != nil {

params/config.go

@@ -617,6 +617,10 @@ func (c *BorConfig) IsDelhi(number *big.Int) bool {
 	return isForked(c.DelhiBlock, number)
 }
 
+func (c *BorConfig) IsSprintStart(number uint64) bool {
+	return number%c.CalculateSprint(number) == 0
+}
+
 func (c *BorConfig) calculateBorConfigHelper(field map[string]uint64, number uint64) uint64 {
 	keys := make([]string, 0, len(field))
 	for k := range field {

rpc/server.go

@@ -23,6 +23,7 @@ import (
 	"sync/atomic"
 
 	mapset "github.com/deckarep/golang-set"
+
 	"github.com/ethereum/go-ethereum/log"
 )
 
@@ -127,9 +128,11 @@ func (s *Server) serveSingleRequest(ctx context.Context, codec ServerCodec) {
 				log.Debug("batch limit %d exceeded: %d requests given", s.BatchLimit, len(reqs))
 			}
 		} else {
+			//nolint:contextcheck
 			h.handleBatch(reqs)
 		}
 	} else {
+		//nolint:contextcheck
 		h.handleMsg(reqs[0])
 	}
 }