forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-28 23:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: get header by block hash

Browse source
This commit is contained in:
fearlessfe 2024-01-19 21:15:07 +08:00 committed by Chen Kai
parent 534b83420e
commit baf8e8aac7
6 changed files with 502 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
go.mod
View file

@ -53,7 +53,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.18
github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
github.com/olekukonko/tablewriter v0.0.5
github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58
github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7
github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7
github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7
@ -71,7 +71,7 @@ require (
golang.org/x/crypto v0.17.0
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
golang.org/x/sync v0.5.0
golang.org/x/sys v0.15.0
golang.org/x/sys v0.16.0
golang.org/x/text v0.14.0
golang.org/x/time v0.3.0
golang.org/x/tools v0.15.0

4
go.sum
View file

@ -494,6 +494,8 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58 h1:EZfd3NpJV+CL5vORquJ2O6eAUjkpI7+ge9a9n9HMyYE=
github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58/go.mod h1:DZ0jYzLzt4ZsCmhI/iqYgGFoNx45OfpEoKzXB8HVALQ=
github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644 h1:vrYEqCVnDS/Z3lLQa+GBrXRtIHN948TWy+aw04O9dpQ=
github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644/go.mod h1:DZ0jYzLzt4ZsCmhI/iqYgGFoNx45OfpEoKzXB8HVALQ=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=
github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
@ -797,6 +799,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

24
p2p/discover/portal_protocol.go
View file

@ -248,7 +248,7 @@ func (p *PortalProtocol) setupUDPListening() (*net.UDPConn, error) {
})
// TODO: ZAP PRODUCTION LOG
logger, err := zap.NewDevelopmentConfig().Build()
logger, err := zap.NewProductionConfig().Build()
if err != nil {
return nil, err
}
@ -1355,7 +1355,7 @@ func (p *PortalProtocol) ContentLookup(contentKey []byte) ([]byte, error) {
lookupContext, cancel := context.WithCancel(context.Background())
defer cancel()
resChan := make(chan []byte, 1)
defer close(resChan)
newLookup(lookupContext, p.table, p.Self().ID(), func(n *node) ([]*node, error) {
return p.contentLookupWorker(unwrapNode(n), contentKey, resChan)
}).run()
@ -1390,6 +1390,26 @@ func (p *PortalProtocol) contentLookupWorker(n *enode.Node, contentKey []byte, r
return wrapedNode, nil
}
func (p *PortalProtocol) ToContentId(contentKey []byte) []byte {
return p.toContentId(contentKey)
}
func (p *PortalProtocol) InRange(contentId []byte) bool {
return inRange(p.Self().ID(), p.nodeRadius, contentId)
}
func (p *PortalProtocol) Get(contentId []byte) ([]byte, error) {
return p.storage.Get(contentId)
}
func (p *PortalProtocol) Put(contentId []byte, content []byte) error {
return p.storage.Put(contentId, content)
}
func (p *PortalProtocol) GetContent() <-chan *ContentElement {
return p.contentQueue
}
func inRange(nodeId enode.ID, nodeRadius *uint256.Int, contentId []byte) bool {
distance := enode.LogDist(nodeId, enode.ID(contentId))
disBig := new(big.Int).SetInt64(int64(distance))

203
portalnetwork/history/history_network.go Normal file
View file

@ -0,0 +1,203 @@
package history
import (
"bytes"
"errors"
"fmt"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
"github.com/ethereum/go-ethereum/rlp"
)
type ContentType byte
const (
BlockHeaderType ContentType = 0x00
BlockBodyType ContentType = 0x01
ReceiptsType ContentType = 0x02
EpochAccumulatorType ContentType = 0x03
)
var (
ErrWithdrawalHashIsNotEqual = errors.New("withdrawals hash is not equal")
ErrTxHashIsNotEqual = errors.New("tx hash is not equal")
ErrUnclesHashIsNotEqual = errors.New("uncles hash is not equal")
ErrReceiptsHashIsNotEqual = errors.New("receipts hash is not equal")
ErrContentOutOfRange = errors.New("content out of range")
ErrHeaderWithProofIsInvalid = errors.New("header proof is invalid")
ErrInvalidBlockHash = errors.New("invalid block hash")
)
type ContentKey struct {
selector ContentType
data []byte
}
func newContentKey(selector ContentType, hash []byte) *ContentKey {
return &ContentKey{
selector: selector,
data: hash,
}
}
func (c *ContentKey) encode() []byte {
res := make([]byte, 0, len(c.data)+1)
res = append(res, byte(c.selector))
res = append(res, c.data...)
return res
}
type HistoryNetwork struct {
portalProtocol *discover.PortalProtocol
masterAccumulator *MasterAccumulator
}
func NewHistoryNetwork(portalProtocol *discover.PortalProtocol, accu *MasterAccumulator) *HistoryNetwork {
return &HistoryNetwork{
portalProtocol: portalProtocol,
masterAccumulator: accu,
}
}
func (h *HistoryNetwork) Start() error {
err := h.portalProtocol.Start()
if err != nil {
return err
}
go h.processContentLoop()
return nil
}
// Currently doing 4 retries on lookups but only when the validation fails.
const requestRetries = 4
func (h *HistoryNetwork) GetBlockHeader(blockHash []byte) (*types.Header, error) {
contentKey := newContentKey(BlockHeaderType, blockHash).encode()
contentId := h.portalProtocol.ToContentId(contentKey)
if !h.portalProtocol.InRange(contentId) {
return nil, ErrContentOutOfRange
}
res, err := h.portalProtocol.Get(contentId)
// other error
if err != nil && err != storage.ErrContentNotFound {
return nil, err
}
// no error
if err == nil {
blockHeaderWithProof, err := DecodeBlockHeaderWithProof(res)
if err != nil {
return nil, err
}
header := new(types.Header)
err = rlp.DecodeBytes(blockHeaderWithProof.Header, header)
return header, err
}
// no content in local storage
for retries := 0; retries < requestRetries; retries++ {
// TODO log the err and continue
content, err := h.portalProtocol.ContentLookup(contentKey)
if err != nil {
continue
}
headerWithProof, err := DecodeBlockHeaderWithProof(content)
if err != nil {
continue
}
header, err := ValidateBlockHeaderBytes(headerWithProof.Header, blockHash)
if err != nil {
continue
}
valid, err := h.verifyHeader(header, *headerWithProof.Proof)
if err != nil || !valid {
continue
}
// TODO handle the error
_ = h.portalProtocol.Put(contentId, content)
return header, nil
}
return nil, storage.ErrContentNotFound
}
func (h *HistoryNetwork) verifyHeader(header *types.Header, proof BlockHeaderProof) (bool, error) {
return h.masterAccumulator.VerifyHeader(*header, proof)
}
func (h *HistoryNetwork) processContentLoop() {
contentChan := h.portalProtocol.GetContent()
for contentElement := range contentChan {
err := h.validateContents(contentElement.ContentKeys, contentElement.Contents)
if err != nil {
continue
}
// TODO gossip the validate content
}
}
func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) error {
switch ContentType(contentKey[0]) {
case BlockHeaderType:
headerWithProof, err := DecodeBlockHeaderWithProof(content)
if err != nil {
return err
}
header, err := ValidateBlockHeaderBytes(headerWithProof.Header, contentKey[1:])
if err != nil {
return err
}
valid, err := h.verifyHeader(header, *headerWithProof.Proof)
if err != nil {
return err
}
if !valid {
return ErrHeaderWithProofIsInvalid
}
return err
case BlockBodyType:
// TODO
case ReceiptsType:
// TODO
case EpochAccumulatorType:
// TODO
}
return errors.New("unknown content type")
}
func (h *HistoryNetwork) validateContents(contentKeys [][]byte, contents [][]byte) error {
for i, content := range contents {
contentKey := contentKeys[i]
err := h.validateContent(contentKey, content)
if err != nil {
return fmt.Errorf("content validate failed with content key %v", contentKey)
}
contentId := h.portalProtocol.ToContentId(contentKey)
_ = h.portalProtocol.Put(contentId, content)
}
return nil
}
func ValidateBlockHeaderBytes(headerBytes []byte, blockHash []byte) (*types.Header, error) {
header := new(types.Header)
err := rlp.DecodeBytes(headerBytes, header)
if err != nil {
return nil, err
}
if header.ExcessBlobGas != nil {
return nil, errors.New("EIP-4844 not yet implemented")
}
hash := header.Hash()
if !bytes.Equal(hash[:], blockHash) {
return nil, ErrInvalidBlockHash
}
return header, nil
}
func DecodeBlockHeaderWithProof(content []byte) (*BlockHeaderWithProof, error) {
headerWithProof := new(BlockHeaderWithProof)
err := headerWithProof.UnmarshalSSZ(content)
return headerWithProof, err
}

257
portalnetwork/history/history_network_test.go Normal file
View file

@ -0,0 +1,257 @@
package history
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"math/big"
"os"
"testing"
"time"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/discover/portalwire"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
)
func ContentId(contentKey []byte) []byte {
digest := sha256.Sum256(contentKey)
return digest[:]
}
// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
func TestContentKey(t *testing.T) {
testCases := []struct {
name string
hash string
contentKey string
contentIdHex string
contentIdU256 string
selector ContentType
}{
{
name: "block header key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
selector: BlockHeaderType,
},
{
name: "block body key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
selector: BlockBodyType,
},
{
name: "receipt key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
selector: ReceiptsType,
},
{
name: "epoch accumelator key",
hash: "e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
contentKey: "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
contentIdHex: "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e",
contentIdU256: "72232402989179419196382321898161638871438419016077939952896528930608027961710",
selector: EpochAccumulatorType,
},
}
for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
hashByte, err := hex.DecodeString(c.hash)
require.NoError(t, err)
contentKey := newContentKey(c.selector, hashByte).encode()
hexKey := hex.EncodeToString(contentKey)
require.Equal(t, hexKey, c.contentKey)
contentId := ContentId(contentKey)
require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
bigNum := big.NewInt(0).SetBytes(contentId)
u256Format, isOverflow := uint256.FromBig(bigNum)
require.False(t, isOverflow)
u256Str := fmt.Sprint(u256Format)
require.Equal(t, u256Str, c.contentIdU256)
})
}
}
func TestValidateHeader(t *testing.T) {
entrys, err := parseBlockHeaderKeyContent()
require.NoError(t, err)
historyNetwork, err := genHistoryNetwork(":7891", nil)
require.NoError(t, err)
for _, entry := range entrys {
err = historyNetwork.validateContent(entry.key, entry.value)
require.NoError(t, err)
headerWithProof, err := DecodeBlockHeaderWithProof(entry.value)
require.NoError(t, err)
// invalid blockhash
_, err = ValidateBlockHeaderBytes(headerWithProof.Header, entry.key)
require.Equal(t, ErrInvalidBlockHash, err)
header, err := ValidateBlockHeaderBytes(headerWithProof.Header, entry.key[1:])
require.NoError(t, err)
// wrong header number
header.Number = big.NewInt(0).Add(header.Number, big.NewInt(122))
valid, err := historyNetwork.verifyHeader(header, *headerWithProof.Proof)
require.False(t, valid)
require.NoError(t, err)
}
}
func TestGetContentByKey(t *testing.T) {
historyNetwork1, err := genHistoryNetwork(":7895", nil)
require.NoError(t, err)
historyNetwork2, err := genHistoryNetwork(":7896", []*enode.Node{historyNetwork1.portalProtocol.Self()})
require.NoError(t, err)
// wait node start
time.Sleep(10 * time.Second)
entryMap, err := parseDataForBlock14764013()
require.NoError(t, err)
headerEntry := entryMap["header"]
// test GetBlockHeader
// no content
header, err := historyNetwork2.GetBlockHeader(headerEntry.key[1:])
require.Error(t, err)
require.Nil(t, header)
contentId := historyNetwork1.portalProtocol.ToContentId(headerEntry.key)
err = historyNetwork1.portalProtocol.Put(contentId, headerEntry.value)
require.NoError(t, err)
// get content from historyNetwork1
header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
require.NoError(t, err)
require.NotNil(t, header)
// get content from local
header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
require.NoError(t, err)
require.NotNil(t, header)
}
type contentEntry struct {
key []byte
value []byte
}
func parseBlockHeaderKeyContent() ([]contentEntry, error) {
headWithProofBytes, err := os.ReadFile("./testdata/header_with_proofs.json")
if err != nil {
return nil, err
}
headerMap := make(map[string]map[string]string)
err = json.Unmarshal(headWithProofBytes, &headerMap)
if err != nil {
return nil, err
}
res := make([]contentEntry, 0)
for _, v := range headerMap {
entry := contentEntry{}
val := v["value"]
bytes, err := hexutil.Decode(val)
if err != nil {
return nil, err
}
entry.value = bytes
key := v["content_key"]
keyBytes, err := hexutil.Decode(key)
if err != nil {
return nil, err
}
entry.key = keyBytes
res = append(res, entry)
}
return res, nil
}
type MockStorage struct {
db map[string][]byte
}
func (m *MockStorage) Get(contentId []byte) ([]byte, error) {
if content, ok := m.db[string(contentId)]; ok {
return content, nil
}
return nil, storage.ErrContentNotFound
}
func (m *MockStorage) Put(contentId []byte, content []byte) error {
m.db[string(contentId)] = content
return nil
}
func genHistoryNetwork(addr string, bootNodes []*enode.Node) (*HistoryNetwork, error) {
conf := discover.DefaultPortalProtocolConfig()
if addr != "" {
conf.ListenAddr = addr
}
if bootNodes != nil {
conf.BootstrapNodes = bootNodes
}
contentQueue := make(chan *discover.ContentElement, 50)
key, err := crypto.GenerateKey()
if err != nil {
panic("couldn't generate key: " + err.Error())
}
portalProtocol, err := discover.NewPortalProtocol(conf, portalwire.HistoryNetwork, key, &MockStorage{db: make(map[string][]byte)}, contentQueue)
if err != nil {
return nil, err
}
accu, err := NewMasterAccumulator()
if err != nil {
return nil, err
}
err = portalProtocol.Start()
if err != nil {
return nil, err
}
return NewHistoryNetwork(portalProtocol, &accu), nil
}
func parseDataForBlock14764013() (map[string]contentEntry, error) {
content, err := os.ReadFile("./testdata/block_14764013.json")
if err != nil {
return nil, err
}
contentMap := make(map[string]map[string]string)
json.Unmarshal(content, &contentMap)
res := make(map[string]contentEntry)
for key, val := range contentMap {
entry := contentEntry{}
contentKey := val["content_key"]
entry.key, err = hexutil.Decode(contentKey)
if err != nil {
return nil, err
}
entry.value, err = hexutil.Decode(val["content_value"])
if err != nil {
return nil, err
}
res[key] = entry
}
return res, nil
}

14
portalnetwork/history/testdata/block_14764013.json vendored Normal file
View file

File diff suppressed because one or more lines are too long