diff --git a/.golangci.yml b/.golangci.yml index b0902cb685..4811690269 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,9 +3,6 @@ run: timeout: 20m tests: true - # default is true. Enables skipping of directories: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs-use-default: true linters: disable-all: true @@ -21,7 +18,7 @@ linters: - staticcheck - bidichk - durationcheck - - exportloopref + - copyloopvar # replacement to exportloopref after go 1.22+ - whitespace - revive # only certain checks enabled @@ -50,6 +47,9 @@ linters-settings: exclude: [""] issues: + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + exclude-dirs-use-default: true exclude-files: - core/genesis_alloc.go exclude-rules: diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 738fb61bce..e2f73e5077 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -44,6 +44,8 @@ import ( ) const ( + defaultSpanLength = 6400 // Default span length i.e. number of bor blocks in a span + zerothSpanEnd = 255 // End block of 0th span checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory inmemorySignatures = 4096 // Number of recent block signatures to keep in memory @@ -224,6 +226,8 @@ type Bor struct { GenesisContractsClient GenesisContract HeimdallClient IHeimdallClient + spanStore SpanStore // Store to save previous span data from heimdall + // The fields below are for testing only fakeDiff bool // Skip difficulty verifications devFakeAuthor bool @@ -257,6 +261,9 @@ func New( recents, _ := lru.NewARC(inmemorySnapshots) signatures, _ := lru.NewARC(inmemorySignatures) + // Create a new span store + spanStore := NewSpanStore(heimdallClient, spanner, chainConfig.ChainID.String()) + c := &Bor{ chainConfig: chainConfig, config: borConfig, @@ -267,6 +274,7 @@ func New( spanner: spanner, GenesisContractsClient: genesisContracts, HeimdallClient: heimdallClient, + spanStore: spanStore, devFakeAuthor: devFakeAuthor, } @@ -467,14 +475,22 @@ func (c *Bor) verifyCascadingFields(chain consensus.ChainHeaderReader, header *t return err } - // Verify the validator list match the local contract - if IsSprintStart(number+1, c.config.CalculateSprint(number)) { - newValidators, err := c.spanner.GetCurrentValidatorsByBlockNrOrHash(context.Background(), rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), number+1) - + // Verify if the producer set in header's extra data matches with the list in span. + // We skip the check for 0th span as the producer set in contract v/s producer set + // in heimdall span is different which will lead a mismatch. Moreover, to make the + // validation stateless, we use the span from heimdall (via span store) instead of + // span from validator set genesis contract as both are supposed to be equivalent. + if number > zerothSpanEnd && IsSprintStart(number+1, c.config.CalculateSprint(number)) { + span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1) if err != nil { return err } + // Use producer set from span as it's equivalent to the data we get from genesis contract + newValidators := make([]*valset.Validator, len(span.SelectedProducers)) + for i, val := range span.SelectedProducers { + newValidators[i] = &val + } sort.Sort(valset.ValidatorsByAddress(newValidators)) headerVals, err := valset.ParseValidators(header.GetValidatorBytes(c.chainConfig)) @@ -561,8 +577,6 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co // at a checkpoint block without a parent (light client CHT), or we have piled // up more headers than allowed to be reorged (chain reinit from a freezer), // consider the checkpoint trusted and snapshot it. - - // TODO fix this // nolint:nestif if number == 0 { checkpoint := chain.GetHeaderByNumber(number) @@ -570,14 +584,14 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co // get checkpoint data hash := checkpoint.Hash() - // get validators and current span - validators, err := c.spanner.GetCurrentValidatorsByHash(context.Background(), hash, number+1) + // get validators from span + span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1) if err != nil { return nil, err } // new snap shot - snap = newSnapshot(c.chainConfig, c.signatures, number, hash, validators) + snap = newSnapshot(c.chainConfig, c.signatures, number, hash, span.ValidatorSet.Validators) if err := snap.store(c.db); err != nil { return nil, err } @@ -1161,7 +1175,7 @@ func (c *Bor) FetchAndCommitSpan( heimdallSpan = *s } else { - response, err := c.HeimdallClient.Span(ctx, newSpanID) + response, err := c.spanStore.spanById(ctx, newSpanID) if err != nil { return err } @@ -1292,6 +1306,8 @@ func validateEventRecord(eventRecord *clerk.EventRecordWithTime, number uint64, func (c *Bor) SetHeimdallClient(h IHeimdallClient) { c.HeimdallClient = h + // Update the heimdall client in span store + c.spanStore.setHeimdallClient(h) } func (c *Bor) GetCurrentValidators(ctx context.Context, headerHash common.Hash, blockNumber uint64) ([]*valset.Validator, error) { diff --git a/consensus/bor/snapshot.go b/consensus/bor/snapshot.go index 796a91ac39..7ae2c489e1 100644 --- a/consensus/bor/snapshot.go +++ b/consensus/bor/snapshot.go @@ -5,7 +5,6 @@ import ( "encoding/json" "github.com/ethereum/go-ethereum/consensus/bor/valset" - "github.com/ethereum/go-ethereum/log" lru "github.com/hashicorp/golang-lru" @@ -161,9 +160,12 @@ func (s *Snapshot) apply(headers []*types.Header, c *Bor) (*Snapshot, error) { v.IncrementProposerPriority(1) if v.CheckEmptyId() { - log.Warn("Empty id found on validator set. Querying on the validatorSet contract") - valsWithId, _ := c.spanner.GetCurrentValidatorsByHash(context.Background(), header.Hash(), number+1) - v.IncludeIds(valsWithId) + // Fetch the validator set from span + span, err := c.spanStore.spanByBlockNumber(context.Background(), number+1) + if err != nil { + return nil, err + } + v.IncludeIds(span.ValidatorSet.Validators) } snap.ValidatorSet = v } diff --git a/consensus/bor/span_store.go b/consensus/bor/span_store.go new file mode 100644 index 0000000000..c7707fe368 --- /dev/null +++ b/consensus/bor/span_store.go @@ -0,0 +1,174 @@ +package bor + +import ( + "context" + "fmt" + + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" + "github.com/ethereum/go-ethereum/consensus/bor/valset" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" +) + +// maxSpanFetchLimit denotes maximum number of future spans to fetch. During snap sync, +// we verify very large batch of headers. The maximum range is not known as of now and +// hence we set a very high limit. It can be reduced later. +const maxSpanFetchLimit = 10_000 + +// SpanStore acts as a simple middleware to cache span data populated from heimdall. It is used +// in multiple places of bor consensus for verification. +type SpanStore struct { + store *lru.ARCCache + + heimdallClient IHeimdallClient + spanner Spanner + + latestKnownSpanId uint64 + chainId string +} + +func NewSpanStore(heimdallClient IHeimdallClient, spanner Spanner, chainId string) SpanStore { + cache, _ := lru.NewARC(10) + return SpanStore{ + store: cache, + heimdallClient: heimdallClient, + spanner: spanner, + latestKnownSpanId: 0, + chainId: chainId, + } +} + +// spanById returns a span given its id. It fetches span from heimdall if not found in cache. +func (s *SpanStore) spanById(ctx context.Context, spanId uint64) (*span.HeimdallSpan, error) { + var currentSpan *span.HeimdallSpan + if value, ok := s.store.Get(spanId); ok { + currentSpan, _ = value.(*span.HeimdallSpan) + } + + if currentSpan == nil { + var err error + if s.heimdallClient == nil { + if spanId == 0 { + currentSpan, err = getMockSpan0(ctx, s.spanner, s.chainId) + if err != nil { + log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err) + return nil, err + } + } else { + return nil, fmt.Errorf("unable to create test span without heimdall client for id %d", spanId) + } + } else { + currentSpan, err = s.heimdallClient.Span(ctx, spanId) + if err != nil { + log.Warn("Unable to fetch span from heimdall", "id", spanId, "err", err) + return nil, err + } + } + s.store.Add(spanId, currentSpan) + if currentSpan.Span.ID > s.latestKnownSpanId { + s.latestKnownSpanId = currentSpan.ID + } + } + + return currentSpan, nil +} + +// spanByBlockNumber returns a span given a block number. It fetches span from heimdall if not found in cache. It +// assumes that a span has been committed before (i.e. is current or past span) and returns an error if +// asked for a future span. This is safe to assume as we don't have a way to find out span id for a future block +// unless we hardcode the span length (which we don't want to). +func (s *SpanStore) spanByBlockNumber(ctx context.Context, blockNumber uint64) (*span.HeimdallSpan, error) { + // As we don't persist latest known span to db, we loose the value on restarts. This leads to multiple heimdall calls + // which can be avoided. Hence we estimate the span id from block number which updates the latest known span id. Note + // that we still check if the block number lies in the range of span before returning it. + estimatedSpanId := estimateSpanId(blockNumber) + // Ignore the return value of this span as we validate it later in the loop + _, err := s.spanById(ctx, estimatedSpanId) + if err != nil { + return nil, err + } + // Iterate over all spans and check for number. This is to replicate the behaviour implemented in + // https://github.com/maticnetwork/genesis-contracts/blob/master/contracts/BorValidatorSet.template#L118-L134 + // This logic is independent of the span length (bit extra effort but maintains equivalence) and will work + // for all span lengths (even if we change it in future). + latestKnownSpanId := s.latestKnownSpanId + for id := int(latestKnownSpanId); id >= 0; id-- { + span, err := s.spanById(ctx, uint64(id)) + if err != nil { + return nil, err + } + if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock { + return span, nil + } + // Check if block number given is out of bounds + if id == int(latestKnownSpanId) && blockNumber > span.EndBlock { + return getFutureSpan(ctx, uint64(id)+1, blockNumber, latestKnownSpanId, s) + } + } + + return nil, fmt.Errorf("span not found for block %d", blockNumber) +} + +// getFutureSpan fetches span for future block number. It is mostly needed during snap sync. +func getFutureSpan(ctx context.Context, id uint64, blockNumber uint64, latestKnownSpanId uint64, s *SpanStore) (*span.HeimdallSpan, error) { + for { + if id > latestKnownSpanId+maxSpanFetchLimit { + return nil, fmt.Errorf("span not found for block %d", blockNumber) + } + span, err := s.spanById(ctx, id) + if err != nil { + return nil, err + } + if blockNumber >= span.StartBlock && blockNumber <= span.EndBlock { + return span, nil + } + id++ + } +} + +// estimateSpanId returns the corresponding span id for the given block number in a deterministic way. +func estimateSpanId(blockNumber uint64) uint64 { + if blockNumber > zerothSpanEnd { + return 1 + (blockNumber-zerothSpanEnd-1)/defaultSpanLength + } + return 0 +} + +// setHeimdallClient sets the underlying heimdall client to be used. It is useful in +// tests where mock heimdall client is set after creation of bor instance explicitly. +func (s *SpanStore) setHeimdallClient(client IHeimdallClient) { + s.heimdallClient = client +} + +// getMockSpan0 constructs a mock span 0 by fetching validator set from genesis state. This should +// only be used in tests where heimdall client is not available. +func getMockSpan0(ctx context.Context, spanner Spanner, chainId string) (*span.HeimdallSpan, error) { + if spanner == nil { + return nil, fmt.Errorf("spanner not available to fetch validator set") + } + + // Fetch validators from genesis state + vals, err := spanner.GetCurrentValidatorsByBlockNrOrHash(ctx, rpc.BlockNumberOrHashWithNumber(0), 0) + if err != nil { + return nil, err + } + validatorSet := valset.ValidatorSet{ + Validators: vals, + Proposer: vals[0], + } + selectedProducers := make([]valset.Validator, len(vals)) + for _, v := range vals { + selectedProducers = append(selectedProducers, *v) + } + return &span.HeimdallSpan{ + Span: span.Span{ + ID: 0, + StartBlock: 0, + EndBlock: 255, + }, + ValidatorSet: validatorSet, + SelectedProducers: selectedProducers, + ChainID: chainId, + }, nil +} diff --git a/consensus/bor/span_store_test.go b/consensus/bor/span_store_test.go new file mode 100644 index 0000000000..b88f2fa24e --- /dev/null +++ b/consensus/bor/span_store_test.go @@ -0,0 +1,220 @@ +package bor + +import ( + "context" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/consensus/bor/clerk" + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint" + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone" + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" + "github.com/stretchr/testify/require" +) + +type MockHeimdallClient struct { +} + +func (h *MockHeimdallClient) Span(ctx context.Context, spanID uint64) (*span.HeimdallSpan, error) { + // Throw error for span id 100 + if spanID == 100 { + return nil, fmt.Errorf("unable to fetch span") + } + + // For everything else, return hardcoded span assuming length 6400 (except for span 0) + if spanID == 0 { + return &span.HeimdallSpan{ + Span: span.Span{ + ID: 0, + StartBlock: 0, + EndBlock: 255, + }, + }, nil + } else { + return &span.HeimdallSpan{ + Span: span.Span{ + ID: spanID, + StartBlock: 6400*(spanID-1) + 256, + EndBlock: 6400*spanID + 255, + }, + }, nil + } +} + +func TestSpanStore_SpanById(t *testing.T) { + spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337") + ctx := context.Background() + + type Testcase struct { + id uint64 + startBlock uint64 + endBlock uint64 + } + + testcases := []Testcase{ + {id: 0, startBlock: 0, endBlock: 255}, + {id: 1, startBlock: 256, endBlock: 6655}, + {id: 2, startBlock: 6656, endBlock: 13055}, + } + + for _, tc := range testcases { + t.Run("", func(t *testing.T) { + span, err := spanStore.spanById(ctx, tc.id) + require.NoError(t, err, "err in spanById for id=%d", tc.id) + require.Equal(t, tc.id, span.ID, "invalid id in spanById for id=%d", tc.id) + require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanById for id=%d", tc.id) + require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanById for id=%d", tc.id) + }) + } + + // Ensure cache is updated + keys := spanStore.store.Keys() + require.Len(t, keys, 3, "invalid length of keys in span store") + + // Ensure latest known span id is updated + require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store") + + // Ask for a few more spans + for i := spanStore.latestKnownSpanId; i <= 20; i++ { + _, err := spanStore.spanById(ctx, i) + require.NoError(t, err, "err in spanById for id=%d", i) + } + + // Ensure cache is updated + keys = spanStore.store.Keys() + require.Len(t, keys, 10, "invalid length of keys in span store") + + // Ensure latest known span id is updated + require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store") + + // Ensure we're still able to fetch old spans even though they're evicted from cache + span, err := spanStore.spanById(ctx, 0) + require.NoError(t, err, "err in spanById after eviction for id=0") + require.Equal(t, uint64(0), span.ID, "invalid id in spanById after eviction for id=0") + require.Equal(t, uint64(0), span.StartBlock, "invalid start block in spanById after eviction for id=0") + require.Equal(t, uint64(255), span.EndBlock, "invalid end block in spanById after eviction for id=0") + + // Try fetching span 100 and ensure error is handled + span, err = spanStore.spanById(ctx, 100) + require.Error(t, err, "expected error in spanById for id=100") + require.Nil(t, span, "expected nil span in spanById for id=100") + + // Ensure latest known span is still the old one + require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store") +} + +func TestSpanStore_SpanByBlockNumber(t *testing.T) { + spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337") + ctx := context.Background() + + type Testcase struct { + blockNumber uint64 + id uint64 + startBlock uint64 + endBlock uint64 + } + + // Insert a few spans + for i := spanStore.latestKnownSpanId; i < 3; i++ { + _, err := spanStore.spanById(ctx, i) + require.NoError(t, err, "err in spanById for id=%d", i) + } + + // Ensure cache is updated + keys := spanStore.store.Keys() + require.Len(t, keys, 3, "invalid length of keys in span store") + + // Ensure latest known span id is updated + require.Equal(t, uint64(2), spanStore.latestKnownSpanId, "invalid latest known span id in span store") + + // Ask for current and past spans via block number + testcases := []Testcase{ + {blockNumber: 0, id: 0, startBlock: 0, endBlock: 255}, + {blockNumber: 1, id: 0, startBlock: 0, endBlock: 255}, + {blockNumber: 255, id: 0, startBlock: 0, endBlock: 255}, + {blockNumber: 256, id: 1, startBlock: 256, endBlock: 6655}, + {blockNumber: 257, id: 1, startBlock: 256, endBlock: 6655}, + {blockNumber: 6000, id: 1, startBlock: 256, endBlock: 6655}, + {blockNumber: 6655, id: 1, startBlock: 256, endBlock: 6655}, + {blockNumber: 6656, id: 2, startBlock: 6656, endBlock: 13055}, + {blockNumber: 10000, id: 2, startBlock: 6656, endBlock: 13055}, + {blockNumber: 13055, id: 2, startBlock: 6656, endBlock: 13055}, + } + + for _, tc := range testcases { + t.Run("", func(t *testing.T) { + span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber) + require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber) + }) + } + + // Insert a few more spans to trigger eviction + for i := spanStore.latestKnownSpanId; i <= 20; i++ { + _, err := spanStore.spanById(ctx, i) + require.NoError(t, err, "err in spanById for id=%d", i) + } + + // Ensure cache is updated + keys = spanStore.store.Keys() + require.Len(t, keys, 10, "invalid length of keys in span store") + + // Ensure latest known span id is updated + require.Equal(t, uint64(20), spanStore.latestKnownSpanId, "invalid latest known span id in span store") + + // Ask for current and past spans + testcases = append(testcases, Testcase{blockNumber: 57856, id: 10, startBlock: 57856, endBlock: 64255}) + testcases = append(testcases, Testcase{blockNumber: 60000, id: 10, startBlock: 57856, endBlock: 64255}) + testcases = append(testcases, Testcase{blockNumber: 64255, id: 10, startBlock: 57856, endBlock: 64255}) + testcases = append(testcases, Testcase{blockNumber: 121856, id: 20, startBlock: 121856, endBlock: 128255}) + testcases = append(testcases, Testcase{blockNumber: 122000, id: 20, startBlock: 121856, endBlock: 128255}) + testcases = append(testcases, Testcase{blockNumber: 128255, id: 20, startBlock: 121856, endBlock: 128255}) + + for _, tc := range testcases { + t.Run("", func(t *testing.T) { + span, err := spanStore.spanByBlockNumber(ctx, tc.blockNumber) + require.NoError(t, err, "err in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.id, span.ID, "invalid id in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.startBlock, span.StartBlock, "invalid start block in spanByBlockNumber for block=%d", tc.blockNumber) + require.Equal(t, tc.endBlock, span.EndBlock, "invalid end block in spanByBlockNumber for block=%d", tc.blockNumber) + }) + } + + // Asking for a future span + span, err := spanStore.spanByBlockNumber(ctx, 128256) // block 128256 belongs to span 21 (future span) + require.NoError(t, err, "err in spanByBlockNumber for future block 128256") + require.Equal(t, uint64(21), span.ID, "invalid id in spanByBlockNumber for future block 128256") + require.Equal(t, uint64(128256), span.StartBlock, "invalid start block in spanByBlockNumber for future block 128256") + require.Equal(t, uint64(134655), span.EndBlock, "invalid end block in spanByBlockNumber for future block 128256") +} + +// Irrelevant to the tests above but necessary for interface compatibility +func (h *MockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchCheckpointCount(ctx context.Context) (int64, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchMilestone(ctx context.Context) (*milestone.Milestone, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchNoAckMilestone(ctx context.Context, milestoneID string) error { + panic("implement me") +} +func (h *MockHeimdallClient) FetchLastNoAckMilestone(ctx context.Context) (string, error) { + panic("implement me") +} +func (h *MockHeimdallClient) FetchMilestoneID(ctx context.Context, milestoneID string) error { + panic("implement me") +} +func (h *MockHeimdallClient) Close() { + panic("implement me") +} diff --git a/miner/fake_miner.go b/miner/fake_miner.go index 2b120cec89..8f7d0b29fe 100644 --- a/miner/fake_miner.go +++ b/miner/fake_miner.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/bor" "github.com/ethereum/go-ethereum/consensus/bor/api" + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" "github.com/ethereum/go-ethereum/consensus/bor/valset" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,17 +47,14 @@ func NewBorDefaultMiner(t *testing.T) *DefaultBorMiner { ethAPI := api.NewMockCaller(ctrl) ethAPI.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + // Mock span 0 for heimdall + span0 := createMockSpanForTest(common.Address{0x1}, "1337") + spanner := bor.NewMockSpanner(ctrl) - spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{ - { - ID: 0, - Address: common.Address{0x1}, - VotingPower: 100, - ProposerPriority: 0, - }, - }, nil).AnyTimes() + spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes() heimdallClient := mocks.NewMockIHeimdallClient(ctrl) + heimdallClient.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes() heimdallClient.EXPECT().Close().Times(1) genesisContracts := bor.NewMockGenesisContract(ctrl) @@ -157,6 +155,32 @@ func NewFakeBor(t TensingObject, chainDB ethdb.Database, chainConfig *params.Cha return bor.New(chainConfig, chainDB, ethAPIMock, spanner, heimdallClientMock, contractMock, false) } +func createMockSpanForTest(address common.Address, chainId string) span.HeimdallSpan { + // Mock span 0 for heimdall calls + validator := valset.Validator{ + ID: 0, + Address: address, + VotingPower: 100, + ProposerPriority: 0, + } + validatorSet := valset.ValidatorSet{ + Validators: []*valset.Validator{&validator}, + Proposer: &validator, + } + span0 := span.HeimdallSpan{ + Span: span.Span{ + ID: 0, + StartBlock: 0, + EndBlock: 255, + }, + ValidatorSet: validatorSet, + SelectedProducers: []valset.Validator{validator}, + ChainID: chainId, + } + + return span0 +} + var ( // Test chain configurations testTxPoolConfigBor legacypool.Config diff --git a/miner/worker_test.go b/miner/worker_test.go index fff63b61b5..b436df265e 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -383,17 +383,14 @@ func getFakeBorFromConfig(t *testing.T, chainConfig *params.ChainConfig) (consen ethAPIMock := api.NewMockCaller(ctrl) ethAPIMock.EXPECT().Call(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + // Mock span 0 for heimdall + span0 := createMockSpanForTest(TestBankAddress, chainConfig.ChainID.String()) + spanner := bor.NewMockSpanner(ctrl) - spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return([]*valset.Validator{ - { - ID: 0, - Address: TestBankAddress, - VotingPower: 100, - ProposerPriority: 0, - }, - }, nil).AnyTimes() + spanner.EXPECT().GetCurrentValidatorsByHash(gomock.Any(), gomock.Any(), gomock.Any()).Return(span0.ValidatorSet.Validators, nil).AnyTimes() heimdallClientMock := mocks.NewMockIHeimdallClient(ctrl) + heimdallClientMock.EXPECT().Span(gomock.Any(), uint64(0)).Return(&span0, nil).AnyTimes() heimdallClientMock.EXPECT().Close().AnyTimes() contractMock := bor.NewMockGenesisContract(ctrl) diff --git a/tests/bor/bor_test.go b/tests/bor/bor_test.go index b3ac2e58c3..5ac8ccf464 100644 --- a/tests/bor/bor_test.go +++ b/tests/bor/bor_test.go @@ -25,8 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/consensus/bor" "github.com/ethereum/go-ethereum/consensus/bor/clerk" - "github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint" - "github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone" "github.com/ethereum/go-ethereum/consensus/bor/valset" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" @@ -40,7 +38,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/tests/bor/mocks" "github.com/ethereum/go-ethereum/triedb" ) @@ -379,39 +376,23 @@ func TestInsertingSpanSizeBlocks(t *testing.T) { chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) + defer _bor.Close() + span0 := createMockSpan(addr, chain.Config().ChainID.String()) _, currentSpan := loadSpanFromFile(t) - h, ctrl := getMockedHeimdallClient(t, currentSpan) - defer func() { - _bor.Close() - ctrl.Finish() - }() - - h.EXPECT().Close().AnyTimes() - - h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{ - Proposer: currentSpan.SelectedProducers[0].Address, - StartBlock: big.NewInt(0), - EndBlock: big.NewInt(int64(spanSize)), - }, nil).AnyTimes() - - h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{ - Proposer: currentSpan.SelectedProducers[0].Address, - StartBlock: big.NewInt(0), - EndBlock: big.NewInt(int64(spanSize)), - }, nil).AnyTimes() - - h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() - - h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() + // Create mock heimdall client + ctrl := gomock.NewController(t) + defer ctrl.Finish() + h := createMockHeimdall(ctrl, &span0, currentSpan) + h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()). + Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes() _bor.SetHeimdallClient(h) block := init.genesis.ToBlock() - // to := int64(block.Header().Time) - currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)} + currentValidators := span0.ValidatorSet.Validators spanner := getMockedSpanner(t, currentValidators) _bor.SetSpanner(spanner) @@ -446,48 +427,29 @@ func TestFetchStateSyncEvents(t *testing.T) { chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) - defer _bor.Close() - // A. Insert blocks for 0th sprint + // Insert blocks for 0th sprint block := init.genesis.ToBlock() - // B.1 Mock /bor/span/1 + // Create a mock span 0 + span0 := createMockSpan(addr, chain.Config().ChainID.String()) + currentValidators := span0.ValidatorSet.Validators + + // Load mock span 0 res, _ := loadSpanFromFile(t) - currentValidators := []*valset.Validator{valset.NewValidator(addr, 10)} - + // reate mock bor spanner spanner := getMockedSpanner(t, currentValidators) _bor.SetSpanner(spanner) - // Insert sprintSize # of blocks so that span is fetched at the start of a new sprint - for i := uint64(1); i < sprintSize; i++ { - if IsSpanEnd(i) { - currentValidators = res.Result.ValidatorSet.Validators - } - - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) - insertNewBlock(t, chain, block) - } - - // B. Before inserting 1st block of the next sprint, mock heimdall deps + // Create mock heimdall client ctrl := gomock.NewController(t) defer ctrl.Finish() - h := mocks.NewMockIHeimdallClient(ctrl) - h.EXPECT().Close().AnyTimes() + h := createMockHeimdall(ctrl, &span0, &res.Result) - h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes() - - h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes() - - h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes() - - h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() - - h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() - - // B.2 Mock State Sync events + // Mock state sync events fromID := uint64(1) // at # sprintSize, events are fetched for [fromID, (block-sprint).Time) to := int64(chain.GetHeaderByNumber(0).Time) @@ -500,6 +462,16 @@ func TestFetchStateSyncEvents(t *testing.T) { h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes() _bor.SetHeimdallClient(h) + // Insert sprintSize # of blocks so that span is fetched at the start of a new sprint + for i := uint64(1); i < sprintSize; i++ { + if IsSpanEnd(i) { + currentValidators = res.Result.ValidatorSet.Validators + } + + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) + insertNewBlock(t, chain, block) + } + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators) // Validate the state sync transactions set by consensus @@ -525,30 +497,24 @@ func TestFetchStateSyncEvents_2(t *testing.T) { chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) - defer _bor.Close() - // Mock /bor/span/1 + // Create a mock span 0 + span0 := createMockSpan(addr, chain.Config().ChainID.String()) + + // Load mock span 1 res, _ := loadSpanFromFile(t) + spanner := getMockedSpanner(t, span0.ValidatorSet.Validators) + _bor.SetSpanner(spanner) + // add the block producer res.Result.ValidatorSet.Validators = append(res.Result.ValidatorSet.Validators, valset.NewValidator(addr, 4500)) ctrl := gomock.NewController(t) defer ctrl.Finish() - h := mocks.NewMockIHeimdallClient(ctrl) - h.EXPECT().Close().AnyTimes() - - h.EXPECT().Span(gomock.Any(), uint64(1)).Return(&res.Result, nil).AnyTimes() - - h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes() - - h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes() - - h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() - - h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() + h := createMockHeimdall(ctrl, &span0, &res.Result) // Mock State Sync events // at # sprintSize, events are fetched for [fromID, (block-sprint).Time) @@ -573,18 +539,9 @@ func TestFetchStateSyncEvents_2(t *testing.T) { // Insert blocks for 0th sprint block := init.genesis.ToBlock() - var currentValidators []*valset.Validator - + // Set the current validators from span0 + currentValidators := span0.ValidatorSet.Validators for i := uint64(1); i <= sprintSize; i++ { - if IsSpanEnd(i) { - currentValidators = res.Result.ValidatorSet.Validators - } else { - currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)} - } - - spanner := getMockedSpanner(t, currentValidators) - _bor.SetSpanner(spanner) - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) insertNewBlock(t, chain, block) } @@ -605,15 +562,22 @@ func TestFetchStateSyncEvents_2(t *testing.T) { h.EXPECT().StateSyncEvents(gomock.Any(), fromID, to).Return(eventRecords, nil).AnyTimes() for i := sprintSize + 1; i <= spanSize; i++ { + // Update the validator set at the end of span and update the respective mocks if IsSpanEnd(i) { currentValidators = res.Result.ValidatorSet.Validators + + // Set the spanner to point to new validator set + spanner := getMockedSpanner(t, currentValidators) + _bor.SetSpanner(spanner) + + // Update the span0's validator set to new validator set. This will be used in verify header when we query + // span to compare validator's set with header's extradata. Even though our span store has old validator set + // stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache. + span0.ValidatorSet.Validators = currentValidators } else { currentValidators = []*valset.Validator{valset.NewValidator(addr, 10)} } - spanner := getMockedSpanner(t, currentValidators) - _bor.SetSpanner(spanner) - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators) insertNewBlock(t, chain, block) } @@ -631,32 +595,25 @@ func TestOutOfTurnSigning(t *testing.T) { chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) - defer _bor.Close() + span0 := createMockSpan(addr, chain.Config().ChainID.String()) + _, heimdallSpan := loadSpanFromFile(t) proposer := valset.NewValidator(addr, 10) heimdallSpan.ValidatorSet.Validators = append(heimdallSpan.ValidatorSet.Validators, proposer) - // add the block producer - h, ctrl := getMockedHeimdallClient(t, heimdallSpan) + ctrl := gomock.NewController(t) defer ctrl.Finish() - h.EXPECT().Close().AnyTimes() - - h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes() - - h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes() - - h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() - - h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() + h := createMockHeimdall(ctrl, &span0, heimdallSpan) + h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()). + Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes() + _bor.SetHeimdallClient(h) spanner := getMockedSpanner(t, heimdallSpan.ValidatorSet.Validators) _bor.SetSpanner(spanner) - _bor.SetHeimdallClient(h) - block := init.genesis.ToBlock() setDifficulty := func(header *types.Header) { @@ -665,8 +622,19 @@ func TestOutOfTurnSigning(t *testing.T) { } } + currentValidators := span0.ValidatorSet.Validators for i := uint64(1); i < spanSize; i++ { - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, heimdallSpan.ValidatorSet.Validators, setDifficulty) + // Update the validator set before sprint end (so that it is returned when called for next block) + // E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification + if i == sprintSize-1 { + currentValidators = heimdallSpan.ValidatorSet.Validators + + // Update the span0's validator set to new validator set. This will be used in verify header when we query + // span to compare validator's set with header's extradata. Even though our span store has old validator set + // stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache. + span0.ValidatorSet.Validators = currentValidators + } + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators, setDifficulty) insertNewBlock(t, chain, block) } @@ -724,27 +692,23 @@ func TestSignerNotFound(t *testing.T) { t.Parallel() log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() _bor := engine.(*bor.Bor) - defer _bor.Close() + span0 := createMockSpan(addr, chain.Config().ChainID.String()) + _, heimdallSpan := loadSpanFromFile(t) - h, ctrl := getMockedHeimdallClient(t, heimdallSpan) + ctrl := gomock.NewController(t) defer ctrl.Finish() - h.EXPECT().Close().AnyTimes() - h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes() - - h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes() - - h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() - - h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() - + h := createMockHeimdall(ctrl, &span0, heimdallSpan) + h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any(), gomock.Any()). + Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes() _bor.SetHeimdallClient(h) block := init.genesis.ToBlock() @@ -777,6 +741,7 @@ func TestSignerNotFound(t *testing.T) { // gasFeeCap - gasTipCap < baseFee. // 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -995,6 +960,7 @@ func TestEIP1559Transition(t *testing.T) { func TestBurnContract(t *testing.T) { t.Parallel() + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -1210,6 +1176,7 @@ func TestBurnContract(t *testing.T) { func TestBurnContractContractFetch(t *testing.T) { t.Parallel() + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) config := params.BorUnittestChainConfig config.Bor.BurntContract = map[string]string{ "10": "0x000000000000000000000000000000000000aaab", @@ -1281,6 +1248,7 @@ func TestBurnContractContractFetch(t *testing.T) { // EIP1559 is not supported without EIP155. An error is expected func TestEIP1559TransitionWithEIP155(t *testing.T) { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -1354,6 +1322,7 @@ func TestEIP1559TransitionWithEIP155(t *testing.T) { // it is up to a user to use protected transactions. so if a transaction is unprotected no errors related to chainID are expected. // transactions are checked in 2 places: transaction pool and blockchain processor. func TestTransitionWithoutEIP155(t *testing.T) { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") @@ -1440,22 +1409,41 @@ func TestTransitionWithoutEIP155(t *testing.T) { func TestJaipurFork(t *testing.T) { t.Parallel() + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + init := buildEthereumInstance(t, rawdb.NewMemoryDatabase()) chain := init.ethereum.BlockChain() engine := init.ethereum.Engine() - _bor := engine.(*bor.Bor) defer _bor.Close() block := init.genesis.ToBlock() + span0 := createMockSpan(addr, chain.Config().ChainID.String()) res, _ := loadSpanFromFile(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + h := createMockHeimdall(ctrl, &span0, &res.Result) + _bor.SetHeimdallClient(h) + spanner := getMockedSpanner(t, res.Result.ValidatorSet.Validators) _bor.SetSpanner(spanner) + currentValidators := span0.ValidatorSet.Validators for i := uint64(1); i < sprintSize; i++ { - block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, res.Result.ValidatorSet.Validators) + // Update the validator set before sprint end (so that it is returned when called for next block) + // E.g. In this case, update on block 3 as snapshot of block 3 will be called for block 4's verification + if i == sprintSize-1 { + currentValidators = res.Result.ValidatorSet.Validators + + // Update the span0's validator set to new validator set. This will be used in verify header when we query + // span to compare validator's set with header's extradata. Even though our span store has old validator set + // stored in cache, we're updating the underlying pointer here and hence we don't need to update the cache. + span0.ValidatorSet.Validators = currentValidators + } + block = buildNextBlock(t, _bor, chain, block, nil, init.genesis.Config.Bor, nil, currentValidators) insertNewBlock(t, chain, block) if block.Number().Uint64() == init.genesis.Config.Bor.JaipurBlock.Uint64()-1 { diff --git a/tests/bor/helper.go b/tests/bor/helper.go index 917d9bf520..2391aec80d 100644 --- a/tests/bor/helper.go +++ b/tests/bor/helper.go @@ -23,6 +23,8 @@ import ( "github.com/ethereum/go-ethereum/consensus/bor" "github.com/ethereum/go-ethereum/consensus/bor/clerk" "github.com/ethereum/go-ethereum/consensus/bor/heimdall" //nolint:typecheck + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint" + "github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone" "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" "github.com/ethereum/go-ethereum/consensus/bor/valset" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" @@ -358,6 +360,46 @@ func getMockedHeimdallClient(t *testing.T, heimdallSpan *span.HeimdallSpan) (*mo return h, ctrl } +func createMockSpan(address common.Address, chainId string) span.HeimdallSpan { + // Mock span 0 for heimdall calls + validator := valset.Validator{ + ID: 0, + Address: address, + VotingPower: 10, + ProposerPriority: 0, + } + validatorSet := valset.ValidatorSet{ + Validators: []*valset.Validator{&validator}, + Proposer: &validator, + } + span0 := span.HeimdallSpan{ + Span: span.Span{ + ID: 0, + StartBlock: 0, + EndBlock: 255, + }, + ValidatorSet: validatorSet, + SelectedProducers: []valset.Validator{validator}, + ChainID: chainId, + } + + return span0 +} + +func createMockHeimdall(ctrl *gomock.Controller, span0, span1 *span.HeimdallSpan) *mocks.MockIHeimdallClient { + h := mocks.NewMockIHeimdallClient(ctrl) + + h.EXPECT().Close().AnyTimes() + h.EXPECT().Span(gomock.Any(), uint64(0)).Return(span0, nil).AnyTimes() + h.EXPECT().Span(gomock.Any(), uint64(1)).Return(span1, nil).AnyTimes() + h.EXPECT().FetchCheckpoint(gomock.Any(), int64(-1)).Return(&checkpoint.Checkpoint{}, nil).AnyTimes() + h.EXPECT().FetchMilestone(gomock.Any()).Return(&milestone.Milestone{}, nil).AnyTimes() + h.EXPECT().FetchLastNoAckMilestone(gomock.Any()).Return("", nil).AnyTimes() + h.EXPECT().FetchNoAckMilestone(gomock.Any(), string("test")).Return(nil).AnyTimes() + + return h +} + func getMockedSpanner(t *testing.T, validators []*valset.Validator) *bor.MockSpanner { t.Helper()