new: New fetch State sync logic

This commit is contained in:
atvanguard 2020-05-17 17:24:09 +05:30
parent 0f9e189a25
commit 868dc81c8a
8 changed files with 302 additions and 149 deletions

View file

@ -41,9 +41,6 @@ import (
"golang.org/x/crypto/sha3"
)
const validatorsetABI = `[{"constant":true,"inputs":[{"name":"span","type":"uint256"}],"name":"getSpan","outputs":[{"name":"number","type":"uint256"},{"name":"startBlock","type":"uint256"},{"name":"endBlock","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"number","type":"uint256"}],"name":"getBorValidators","outputs":[{"name":"","type":"address[]"},{"name":"","type":"uint256[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"span","type":"uint256"},{"name":"signer","type":"address"}],"name":"isProducer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newSpan","type":"uint256"},{"name":"startBlock","type":"uint256"},{"name":"endBlock","type":"uint256"},{"name":"validatorBytes","type":"bytes"},{"name":"producerBytes","type":"bytes"}],"name":"commitSpan","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"span","type":"uint256"},{"name":"signer","type":"address"}],"name":"isValidator","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"proposeSpan","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"currentSpanNumber","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getNextSpan","outputs":[{"name":"number","type":"uint256"},{"name":"startBlock","type":"uint256"},{"name":"endBlock","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getInitialValidators","outputs":[{"name":"","type":"address[]"},{"name":"","type":"uint256[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"spanProposalPending","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getCurrentSpan","outputs":[{"name":"number","type":"uint256"},{"name":"startBlock","type":"uint256"},{"name":"endBlock","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"number","type":"uint256"}],"name":"getSpanByBlock","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"","type":"address[]"},{"name":"","type":"uint256[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"vote","type":"bytes"},{"name":"sigs","type":"bytes"},{"name":"txBytes","type":"bytes"},{"name":"proof","type":"bytes"}],"name":"validateValidatorSet","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
const stateReceiverABI = `[{"constant":true,"inputs":[{"name":"","type":"uint256"}],"name":"states","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"recordBytes","type":"bytes"}],"name":"commitState","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getPendingStates","outputs":[{"name":"","type":"uint256[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"SYSTEM_ADDRESS","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"validatorSet","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"vote","type":"bytes"},{"name":"sigs","type":"bytes"},{"name":"txBytes","type":"bytes"},{"name":"proof","type":"bytes"}],"name":"validateValidatorSet","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"isValidatorSetContract","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"stateId","type":"uint256"}],"name":"proposeState","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"signer","type":"address"}],"name":"isProducer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"signer","type":"address"}],"name":"isValidator","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"}]`
const (
checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database
inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory
@ -66,6 +63,7 @@ var (
validatorHeaderBytesLength = common.AddressLength + 20 // address + power
systemAddress = common.HexToAddress("0xffffFFFfFFffffffffffffffFfFFFfffFFFfFFfE")
stateFetchLimit = 50
)
// Various error messages to mark blocks invalid. These should be private to
@ -233,10 +231,11 @@ type Bor struct {
signFn SignerFn // Signer function to authorize hashes with
lock sync.RWMutex // Protects the signer fields
ethAPI *ethapi.PublicBlockChainAPI
validatorSetABI abi.ABI
stateReceiverABI abi.ABI
HeimdallClient IHeimdallClient
ethAPI *ethapi.PublicBlockChainAPI
genesisContractsClient *GenesisContractsClient
validatorSetABI abi.ABI
stateReceiverABI abi.ABI
HeimdallClient IHeimdallClient
stateDataFeed event.Feed
scope event.SubscriptionScope
@ -266,17 +265,18 @@ func New(
vABI, _ := abi.JSON(strings.NewReader(validatorsetABI))
sABI, _ := abi.JSON(strings.NewReader(stateReceiverABI))
heimdallClient, _ := NewHeimdallClient(heimdallURL)
genesisContractsClient := NewGenesisContractsClient(chainConfig, borConfig.ValidatorContract, borConfig.StateReceiverContract, ethAPI)
c := &Bor{
chainConfig: chainConfig,
config: borConfig,
db: db,
ethAPI: ethAPI,
recents: recents,
signatures: signatures,
validatorSetABI: vABI,
stateReceiverABI: sABI,
HeimdallClient: heimdallClient,
chainConfig: chainConfig,
config: borConfig,
db: db,
ethAPI: ethAPI,
recents: recents,
signatures: signatures,
validatorSetABI: vABI,
stateReceiverABI: sABI,
genesisContractsClient: genesisContractsClient,
HeimdallClient: heimdallClient,
}
return c
@ -703,7 +703,7 @@ func (c *Bor) FinalizeAndAssemble(chain consensus.ChainReader, header *types.Hea
// commit statees
if err := c.CommitStates(state, header, cx); err != nil {
log.Error("Error while committing states", "error", err)
// return nil, err
return nil, err
}
}
@ -867,38 +867,6 @@ func (c *Bor) Close() error {
return nil
}
// Checks if "force" proposeSpan has been set
func (c *Bor) isSpanPending(snapshotNumber uint64) (bool, error) {
blockNr := rpc.BlockNumber(snapshotNumber)
method := "spanProposalPending"
// get packed data
data, err := c.validatorSetABI.Pack(method)
if err != nil {
log.Error("Unable to pack tx for spanProposalPending", "error", err)
return false, err
}
msgData := (hexutil.Bytes)(data)
toAddress := common.HexToAddress(c.config.ValidatorContract)
gas := (hexutil.Uint64)(uint64(math.MaxUint64 / 2))
result, err := c.ethAPI.Call(context.Background(), ethapi.CallArgs{
Gas: &gas,
To: &toAddress,
Data: &msgData,
}, blockNr)
if err != nil {
return false, err
}
var ret0 = new(bool)
if err := c.validatorSetABI.Unpack(ret0, method, result); err != nil {
return false, err
}
return *ret0, nil
}
// GetCurrentSpan get current span from contract
func (c *Bor) GetCurrentSpan(snapshotNumber uint64) (*Span, error) {
// block
@ -1003,37 +971,14 @@ func (c *Bor) checkAndCommitSpan(
chain core.ChainContext,
) error {
headerNumber := header.Number.Uint64()
pending := false
var span *Span = nil
errors := make(chan error)
go func() {
var err error
pending, err = c.isSpanPending(headerNumber - 1)
errors <- err
}()
go func() {
var err error
span, err = c.GetCurrentSpan(headerNumber - 1)
errors <- err
}()
var err error
for i := 0; i < 2; i++ {
err = <-errors
if err != nil {
close(errors)
return err
}
span, err := c.GetCurrentSpan(headerNumber - 1)
if err != nil {
return err
}
close(errors)
// commit span if there is new span pending or span is ending or end block is not set
if pending || c.needToCommitSpan(span, headerNumber) {
if c.needToCommitSpan(span, headerNumber) {
err := c.fetchAndCommitSpan(span.ID+1, state, header, chain)
return err
}
return nil
}
@ -1170,83 +1115,87 @@ func (c *Bor) GetPendingStateProposals(snapshotNumber uint64) ([]*big.Int, error
func (c *Bor) CommitStates(
state *state.StateDB,
header *types.Header,
chain core.ChainContext,
chain chainContext,
) error {
// get pending state proposals
stateIds, err := c.GetPendingStateProposals(header.Number.Uint64() - 1)
number := header.Number.Uint64()
if number < c.config.Sprint {
return errors.New("Requested to commit states too soon")
}
lastSync, err := c.genesisContractsClient.LastStateSyncTime(number - 1)
if err != nil {
return err
}
from := lastSync.Add(time.Second * 1) // querying the interval [from, to)
to := time.Unix(int64(chain.Chain.GetHeaderByNumber(number-1).Time), 0)
log.Info(
"Fetching state updates from Heimdall",
"from", from.Format(time.RFC3339),
"to", to.Format(time.RFC3339))
// state ids
if len(stateIds) > 0 {
log.Debug("Found new proposed states", "numberOfStates", len(stateIds))
}
method := "commitState"
// itereate through state ids
for _, stateID := range stateIds {
// fetch from heimdall
response, err := c.HeimdallClient.FetchWithRetry("clerk", "event-record", strconv.FormatUint(stateID.Uint64(), 10))
page := 1
eventRecords := make([]*EventRecordWithTime, 0)
for {
query := fmt.Sprintf("clerk/event-record/list?from-time=%d&to-time=%d&page=%d&limit=%d", from.Unix(), to.Unix(), page, stateFetchLimit)
log.Info("Fetching state events", "query", query)
response, err := c.HeimdallClient.FetchWithRetry(query)
if err != nil {
return err
}
// get event record
var eventRecord EventRecord
if err := json.Unmarshal(response.Result, &eventRecord); err != nil {
var _eventRecords []*EventRecordWithTime
if err := json.Unmarshal(response.Result, &_eventRecords); err != nil {
return err
}
eventRecords = append(eventRecords, _eventRecords...)
if len(_eventRecords) < stateFetchLimit {
break
}
page++
}
sort.SliceStable(eventRecords, func(i, j int) bool {
return eventRecords[i].Time.Before(eventRecords[j].Time)
})
// check if chain id matches with event record
if eventRecord.ChainID != c.chainConfig.ChainID.String() {
return fmt.Errorf(
"Chain id proposed state in span, %s, and bor chain id, %s, doesn't match",
eventRecord.ChainID,
c.chainConfig.ChainID,
)
chainID := c.chainConfig.ChainID.String()
for _, eventRecord := range eventRecords {
// if chanId doesn't match with the record, simply ignore it
if eventRecord.ChainID != chainID {
log.Error(fmt.Sprintln("Chain Id in received event %s and bor chain Id %s, don't match", eventRecord, chainID))
continue
}
// validateEventRecord checks whether an event lies in the specified time range
// since the events are sorted by time and if it turns out that event i lies outside the time range,
// it would mean all subsequent events lie outside of the time range. Hence we don't probe any further and break the loop
if err := validateEventRecord(eventRecord, number, from, to); err != nil {
log.Error(
fmt.Sprintf(
"Received event %s does not lie in the time range, from %s, to %s",
eventRecord, from.Format(time.RFC3339), to.Format(time.RFC3339)))
break
}
log.Info("→ committing new state",
"id", eventRecord.ID,
"contract", eventRecord.Contract,
"data", hex.EncodeToString(eventRecord.Data),
"txHash", eventRecord.TxHash,
"chainID", eventRecord.ChainID,
)
stateData := types.StateData{
Did: eventRecord.ID,
Contract: eventRecord.Contract,
Data: hex.EncodeToString(eventRecord.Data),
TxHash: eventRecord.TxHash,
}
go func() {
c.stateDataFeed.Send(core.NewStateChangeEvent{StateData: &stateData})
}()
recordBytes, err := rlp.EncodeToBytes(eventRecord)
if err != nil {
return err
}
// get packed data for commit state
data, err := c.stateReceiverABI.Pack(method, recordBytes)
if err != nil {
log.Error("Unable to pack tx for commitState", "error", err)
return err
}
// get system message
msg := getSystemMessage(common.HexToAddress(c.config.StateReceiverContract), data)
// apply message
if err := applyMessage(msg, state, header, c.chainConfig, chain); err != nil {
if err := c.genesisContractsClient.CommitState(eventRecord, state, header, chain); err != nil {
return err
}
}
return nil
}
func validateEventRecord(eventRecord *EventRecordWithTime, number uint64, from, to time.Time) error {
inRange := (eventRecord.Time.Equal(from) || eventRecord.Time.After(from)) && eventRecord.Time.Before(to)
if !inRange {
return &InvalidStateReceivedError{number, &from, &to, eventRecord}
}
return nil
}

View file

@ -2,6 +2,9 @@ package bortest
import (
"encoding/hex"
"fmt"
// "encoding/json"
// "fmt"
"math/big"
"testing"
@ -10,6 +13,7 @@ import (
"github.com/maticnetwork/bor/core/rawdb"
"github.com/maticnetwork/bor/crypto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/maticnetwork/bor/core/types"
@ -21,26 +25,27 @@ func TestCommitSpan(t *testing.T) {
chain := init.ethereum.BlockChain()
engine := init.ethereum.Engine()
_bor := engine.(*bor.Bor)
// Mock HeimdallClient.FetchWithRetry to return span data from span.json
res, heimdallSpan := loadSpanFromFile(t)
h := &mocks.IHeimdallClient{}
h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
h, heimdallSpan := getMockedHeimdallClient(t)
_bor.SetHeimdallClient(h)
db := init.ethereum.ChainDb()
block := init.genesis.ToBlock(db)
var to int64
// Insert sprintSize # of blocks so that span is fetched at the start of a new sprint
for i := uint64(1); i <= sprintSize; i++ {
block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor)
insertNewBlock(t, chain, block)
if i == sprintSize-1 {
// at # sprintSize, events are fetched for the internal [from, (block-1).Time)
to = int64(block.Header().Time)
}
}
// FetchWithRetry is invoked 2 times
// 1. bor.FinalizeAndAssemble to prepare a new block when calling buildNextBlock
// 2. bor.Finalize via(bc.insertChain => bc.processor.Process)
assert.True(t, h.AssertCalled(t, "FetchWithRetry", "bor", "span", "1"))
validators, err := _bor.GetCurrentValidators(sprintSize, 256) // new span starts at 256
query := fmt.Sprintf("clerk/event-record/list?from-time=%d&to-time=%d&page=1&limit=50", 1, to)
assert.True(t, h.AssertCalled(t, "FetchWithRetry", query))
validators, err := _bor.GetCurrentValidators(sprintSize, spanSize) // new span starts at 256
if err != nil {
t.Fatalf("%s", err)
}
@ -57,6 +62,8 @@ func TestIsValidatorAction(t *testing.T) {
chain := init.ethereum.BlockChain()
engine := init.ethereum.Engine()
_bor := engine.(*bor.Bor)
h, heimdallSpan := getMockedHeimdallClient(t)
_bor.SetHeimdallClient(h)
proposeStateData, _ := hex.DecodeString("ede01f170000000000000000000000000000000000000000000000000000000000000000")
proposeSpanData, _ := hex.DecodeString("4b0e4d17")
@ -77,11 +84,6 @@ func TestIsValidatorAction(t *testing.T) {
)
assert.True(t, _bor.IsValidatorAction(chain, addr, tx))
res, heimdallSpan := loadSpanFromFile(t)
h := &mocks.IHeimdallClient{}
h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
_bor.SetHeimdallClient(h)
db := init.ethereum.ChainDb()
block := init.genesis.ToBlock(db)
@ -115,10 +117,7 @@ func TestOutOfTurnSigning(t *testing.T) {
chain := init.ethereum.BlockChain()
engine := init.ethereum.Engine()
_bor := engine.(*bor.Bor)
res, _ := loadSpanFromFile(t)
h := &mocks.IHeimdallClient{}
h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
h, _ := getMockedHeimdallClient(t)
_bor.SetHeimdallClient(h)
db := init.ethereum.ChainDb()
@ -160,3 +159,14 @@ func TestOutOfTurnSigning(t *testing.T) {
_, err = chain.InsertChain([]*types.Block{block})
assert.Nil(t, err)
}
func getMockedHeimdallClient(t *testing.T) (*mocks.IHeimdallClient, *bor.HeimdallSpan) {
res, heimdallSpan := loadSpanFromFile(t)
h := &mocks.IHeimdallClient{}
h.On("FetchWithRetry", "bor", "span", "1").Return(res, nil)
res = zeroResultPayload(t)
// query := fmt.Sprintf("clerk/event-record/list?from-time=%d&to-time=%d&page=1&limit=50", 1, 1589709047)
h.On("FetchWithRetry", mock.AnythingOfType("string")).Return(res, nil)
return h, heimdallSpan
}

File diff suppressed because one or more lines are too long

View file

@ -156,6 +156,18 @@ func sign(t *testing.T, header *types.Header, signer []byte) {
copy(header.Extra[len(header.Extra)-extraSeal:], sig)
}
func zeroResultPayload(t *testing.T) *bor.ResponseWithHeight {
stateData, err := ioutil.ReadFile("states.json")
if err != nil {
t.Fatalf("%s", err)
}
res := &bor.ResponseWithHeight{}
if err := json.Unmarshal(stateData, res); err != nil {
t.Fatalf("%s", err)
}
return res
}
func loadSpanFromFile(t *testing.T) (*bor.ResponseWithHeight, *bor.HeimdallSpan) {
spanData, err := ioutil.ReadFile("span.json")
if err != nil {

View file

@ -0,0 +1,23 @@
{
"height": "0",
"result": [
{
"id": 1,
"contract": "0xb55969a6d60413a63291a1de572269875df541e3",
"data": "0x00000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f6500000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f65000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000014",
"tx_hash": "0x7b113e09d98b6d4be1dedfbc0746e34876de767f2cb8b58ff00160a160811dd6",
"log_index": 0,
"bor_chain_id": "15001",
"record_time": "2020-05-15T13:36:38.580995Z"
},
{
"id": 2,
"contract": "0xb55969a6d60413a63291a1de572269875df541e3",
"data": "0x00000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f6500000000000000000000000048aa8d4af32551892fcf08ad63be7dd206d46f65000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000015",
"tx_hash": "0xb72358aff8e4d61f4de97a37a40ddda986c081e0de8036e0a78c4b61b067cba9",
"log_index": 0,
"bor_chain_id": "15001",
"record_time": "2020-05-15T13:42:37.319058Z"
}
]
}

View file

@ -1,6 +1,9 @@
package bor
import (
"fmt"
"time"
"github.com/maticnetwork/bor/common"
"github.com/maticnetwork/bor/common/hexutil"
)
@ -14,3 +17,33 @@ type EventRecord struct {
LogIndex uint64 `json:"log_index" yaml:"log_index"`
ChainID string `json:"bor_chain_id" yaml:"bor_chain_id"`
}
type EventRecordWithTime struct {
EventRecord
Time time.Time `json:"record_time" yaml:"record_time"`
}
// String returns the string representatin of span
func (e *EventRecordWithTime) String() string {
return fmt.Sprintf(
"id %v, contract %v, data: %v, txHash: %v, logIndex: %v, chainId: %v, time %s",
e.ID,
e.Contract.String(),
e.Data.String(),
e.TxHash.Hex(),
e.LogIndex,
e.ChainID,
e.Time.Format(time.RFC3339),
)
}
func (e *EventRecordWithTime) BuildEventRecord() *EventRecord {
return &EventRecord{
ID: e.ID,
Contract: e.Contract,
Data: e.Data,
TxHash: e.TxHash,
LogIndex: e.LogIndex,
ChainID: e.ChainID,
}
}

View file

@ -2,6 +2,7 @@ package bor
import (
"fmt"
"time"
"github.com/maticnetwork/bor/common"
)
@ -142,3 +143,20 @@ func (e *WrongDifficultyError) Error() string {
e.Signer,
)
}
type InvalidStateReceivedError struct {
Number uint64
From *time.Time
To *time.Time
Event *EventRecordWithTime
}
func (e *InvalidStateReceivedError) Error() string {
return fmt.Sprintf(
"Received event with invalid timestamp at block %d. Requested events from %s to %s. Received %s\n",
e.Number,
e.From.Format(time.RFC3339),
e.To.Format(time.RFC3339),
e.Event,
)
}

File diff suppressed because one or more lines are too long