feat: get header by block hash

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58
+	github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7
 	github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7
@@ -71,7 +71,7 @@ require (
 	golang.org/x/crypto v0.17.0
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.15.0
+	golang.org/x/sys v0.16.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.3.0
 	golang.org/x/tools v0.15.0

go.sum

@@ -494,6 +494,8 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58 h1:EZfd3NpJV+CL5vORquJ2O6eAUjkpI7+ge9a9n9HMyYE=
 github.com/optimism-java/utp-go v0.0.0-20231225095152-5a9690d82b58/go.mod h1:DZ0jYzLzt4ZsCmhI/iqYgGFoNx45OfpEoKzXB8HVALQ=
+github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644 h1:vrYEqCVnDS/Z3lLQa+GBrXRtIHN948TWy+aw04O9dpQ=
+github.com/optimism-java/utp-go v0.0.0-20240117090415-3a5aad17f644/go.mod h1:DZ0jYzLzt4ZsCmhI/iqYgGFoNx45OfpEoKzXB8HVALQ=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
@@ -797,6 +799,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

p2p/discover/portal_protocol.go

@@ -248,7 +248,7 @@ func (p *PortalProtocol) setupUDPListening() (*net.UDPConn, error) {
 		})
 
 	// TODO: ZAP PRODUCTION LOG
-	logger, err := zap.NewDevelopmentConfig().Build()
+	logger, err := zap.NewProductionConfig().Build()
 	if err != nil {
 		return nil, err
 	}
@@ -1355,7 +1355,7 @@ func (p *PortalProtocol) ContentLookup(contentKey []byte) ([]byte, error) {
 	lookupContext, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	resChan := make(chan []byte, 1)
-
+	defer close(resChan)
 	newLookup(lookupContext, p.table, p.Self().ID(), func(n *node) ([]*node, error) {
 		return p.contentLookupWorker(unwrapNode(n), contentKey, resChan)
 	}).run()
@@ -1390,6 +1390,26 @@ func (p *PortalProtocol) contentLookupWorker(n *enode.Node, contentKey []byte, r
 	return wrapedNode, nil
 }
 
+func (p *PortalProtocol) ToContentId(contentKey []byte) []byte {
+	return p.toContentId(contentKey)
+}
+
+func (p *PortalProtocol) InRange(contentId []byte) bool {
+	return inRange(p.Self().ID(), p.nodeRadius, contentId)
+}
+
+func (p *PortalProtocol) Get(contentId []byte) ([]byte, error) {
+	return p.storage.Get(contentId)
+}
+
+func (p *PortalProtocol) Put(contentId []byte, content []byte) error {
+	return p.storage.Put(contentId, content)
+}
+
+func (p *PortalProtocol) GetContent() <-chan *ContentElement {
+	return p.contentQueue
+}
+
 func inRange(nodeId enode.ID, nodeRadius *uint256.Int, contentId []byte) bool {
 	distance := enode.LogDist(nodeId, enode.ID(contentId))
 	disBig := new(big.Int).SetInt64(int64(distance))

portalnetwork/history/history_network.go

@@ -0,0 +1,203 @@
+package history
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/portalnetwork/storage"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+type ContentType byte
+
+const (
+	BlockHeaderType      ContentType = 0x00
+	BlockBodyType        ContentType = 0x01
+	ReceiptsType         ContentType = 0x02
+	EpochAccumulatorType ContentType = 0x03
+)
+
+var (
+	ErrWithdrawalHashIsNotEqual = errors.New("withdrawals hash is not equal")
+	ErrTxHashIsNotEqual         = errors.New("tx hash is not equal")
+	ErrUnclesHashIsNotEqual     = errors.New("uncles hash is not equal")
+	ErrReceiptsHashIsNotEqual   = errors.New("receipts hash is not equal")
+	ErrContentOutOfRange        = errors.New("content out of range")
+	ErrHeaderWithProofIsInvalid = errors.New("header proof is invalid")
+	ErrInvalidBlockHash         = errors.New("invalid block hash")
+)
+
+type ContentKey struct {
+	selector ContentType
+	data     []byte
+}
+
+func newContentKey(selector ContentType, hash []byte) *ContentKey {
+	return &ContentKey{
+		selector: selector,
+		data:     hash,
+	}
+}
+
+func (c *ContentKey) encode() []byte {
+	res := make([]byte, 0, len(c.data)+1)
+	res = append(res, byte(c.selector))
+	res = append(res, c.data...)
+	return res
+}
+
+type HistoryNetwork struct {
+	portalProtocol    *discover.PortalProtocol
+	masterAccumulator *MasterAccumulator
+}
+
+func NewHistoryNetwork(portalProtocol *discover.PortalProtocol, accu *MasterAccumulator) *HistoryNetwork {
+	return &HistoryNetwork{
+		portalProtocol:    portalProtocol,
+		masterAccumulator: accu,
+	}
+}
+
+func (h *HistoryNetwork) Start() error {
+	err := h.portalProtocol.Start()
+	if err != nil {
+		return err
+	}
+	go h.processContentLoop()
+	return nil
+}
+
+// Currently doing 4 retries on lookups but only when the validation fails.
+const requestRetries = 4
+
+func (h *HistoryNetwork) GetBlockHeader(blockHash []byte) (*types.Header, error) {
+	contentKey := newContentKey(BlockHeaderType, blockHash).encode()
+	contentId := h.portalProtocol.ToContentId(contentKey)
+	if !h.portalProtocol.InRange(contentId) {
+		return nil, ErrContentOutOfRange
+	}
+
+	res, err := h.portalProtocol.Get(contentId)
+	// other error
+	if err != nil && err != storage.ErrContentNotFound {
+		return nil, err
+	}
+	// no error
+	if err == nil {
+		blockHeaderWithProof, err := DecodeBlockHeaderWithProof(res)
+		if err != nil {
+			return nil, err
+		}
+		header := new(types.Header)
+		err = rlp.DecodeBytes(blockHeaderWithProof.Header, header)
+		return header, err
+	}
+	// no content in local storage
+	for retries := 0; retries < requestRetries; retries++ {
+		// TODO log the err and continue
+		content, err := h.portalProtocol.ContentLookup(contentKey)
+		if err != nil {
+			continue
+		}
+
+		headerWithProof, err := DecodeBlockHeaderWithProof(content)
+		if err != nil {
+			continue
+		}
+
+		header, err := ValidateBlockHeaderBytes(headerWithProof.Header, blockHash)
+		if err != nil {
+			continue
+		}
+		valid, err := h.verifyHeader(header, *headerWithProof.Proof)
+		if err != nil || !valid {
+			continue
+		}
+		// TODO handle the error
+		_ = h.portalProtocol.Put(contentId, content)
+		return header, nil
+	}
+	return nil, storage.ErrContentNotFound
+}
+
+func (h *HistoryNetwork) verifyHeader(header *types.Header, proof BlockHeaderProof) (bool, error) {
+	return h.masterAccumulator.VerifyHeader(*header, proof)
+}
+
+func (h *HistoryNetwork) processContentLoop() {
+	contentChan := h.portalProtocol.GetContent()
+	for contentElement := range contentChan {
+		err := h.validateContents(contentElement.ContentKeys, contentElement.Contents)
+		if err != nil {
+			continue
+		}
+		// TODO gossip the validate content
+	}
+}
+
+func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) error {
+	switch ContentType(contentKey[0]) {
+	case BlockHeaderType:
+		headerWithProof, err := DecodeBlockHeaderWithProof(content)
+		if err != nil {
+			return err
+		}
+		header, err := ValidateBlockHeaderBytes(headerWithProof.Header, contentKey[1:])
+		if err != nil {
+			return err
+		}
+		valid, err := h.verifyHeader(header, *headerWithProof.Proof)
+		if err != nil {
+			return err
+		}
+		if !valid {
+			return ErrHeaderWithProofIsInvalid
+		}
+		return err
+	case BlockBodyType:
+		// TODO
+	case ReceiptsType:
+		// TODO
+	case EpochAccumulatorType:
+		// TODO
+	}
+	return errors.New("unknown content type")
+}
+
+func (h *HistoryNetwork) validateContents(contentKeys [][]byte, contents [][]byte) error {
+	for i, content := range contents {
+		contentKey := contentKeys[i]
+		err := h.validateContent(contentKey, content)
+		if err != nil {
+			return fmt.Errorf("content validate failed with content key %v", contentKey)
+		}
+		contentId := h.portalProtocol.ToContentId(contentKey)
+		_ = h.portalProtocol.Put(contentId, content)
+	}
+	return nil
+}
+
+func ValidateBlockHeaderBytes(headerBytes []byte, blockHash []byte) (*types.Header, error) {
+	header := new(types.Header)
+	err := rlp.DecodeBytes(headerBytes, header)
+	if err != nil {
+		return nil, err
+	}
+	if header.ExcessBlobGas != nil {
+		return nil, errors.New("EIP-4844 not yet implemented")
+	}
+	hash := header.Hash()
+	if !bytes.Equal(hash[:], blockHash) {
+		return nil, ErrInvalidBlockHash
+	}
+	return header, nil
+}
+
+func DecodeBlockHeaderWithProof(content []byte) (*BlockHeaderWithProof, error) {
+	headerWithProof := new(BlockHeaderWithProof)
+	err := headerWithProof.UnmarshalSSZ(content)
+	return headerWithProof, err
+}

portalnetwork/history/history_network_test.go

@@ -0,0 +1,257 @@
+package history
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/discover/portalwire"
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/portalnetwork/storage"
+	"github.com/holiman/uint256"
+	"github.com/stretchr/testify/require"
+)
+
+func ContentId(contentKey []byte) []byte {
+	digest := sha256.Sum256(contentKey)
+	return digest[:]
+}
+
+// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
+func TestContentKey(t *testing.T) {
+	testCases := []struct {
+		name          string
+		hash          string
+		contentKey    string
+		contentIdHex  string
+		contentIdU256 string
+		selector      ContentType
+	}{
+		{
+			name:          "block header key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
+			contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
+			selector:      BlockHeaderType,
+		},
+		{
+			name:          "block body key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
+			contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
+			selector:      BlockBodyType,
+		},
+		{
+			name:          "receipt key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
+			contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
+			selector:      ReceiptsType,
+		},
+		{
+			name:          "epoch accumelator key",
+			hash:          "e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
+			contentKey:    "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
+			contentIdHex:  "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e",
+			contentIdU256: "72232402989179419196382321898161638871438419016077939952896528930608027961710",
+			selector:      EpochAccumulatorType,
+		},
+	}
+
+	for _, c := range testCases {
+		t.Run(c.name, func(t *testing.T) {
+			hashByte, err := hex.DecodeString(c.hash)
+			require.NoError(t, err)
+
+			contentKey := newContentKey(c.selector, hashByte).encode()
+			hexKey := hex.EncodeToString(contentKey)
+			require.Equal(t, hexKey, c.contentKey)
+			contentId := ContentId(contentKey)
+			require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
+
+			bigNum := big.NewInt(0).SetBytes(contentId)
+			u256Format, isOverflow := uint256.FromBig(bigNum)
+			require.False(t, isOverflow)
+			u256Str := fmt.Sprint(u256Format)
+			require.Equal(t, u256Str, c.contentIdU256)
+		})
+	}
+}
+
+func TestValidateHeader(t *testing.T) {
+	entrys, err := parseBlockHeaderKeyContent()
+	require.NoError(t, err)
+	historyNetwork, err := genHistoryNetwork(":7891", nil)
+	require.NoError(t, err)
+	for _, entry := range entrys {
+		err = historyNetwork.validateContent(entry.key, entry.value)
+		require.NoError(t, err)
+
+		headerWithProof, err := DecodeBlockHeaderWithProof(entry.value)
+		require.NoError(t, err)
+		// invalid blockhash
+		_, err = ValidateBlockHeaderBytes(headerWithProof.Header, entry.key)
+		require.Equal(t, ErrInvalidBlockHash, err)
+		header, err := ValidateBlockHeaderBytes(headerWithProof.Header, entry.key[1:])
+		require.NoError(t, err)
+		// wrong header number
+		header.Number = big.NewInt(0).Add(header.Number, big.NewInt(122))
+		valid, err := historyNetwork.verifyHeader(header, *headerWithProof.Proof)
+		require.False(t, valid)
+		require.NoError(t, err)
+	}
+}
+
+func TestGetContentByKey(t *testing.T) {
+	historyNetwork1, err := genHistoryNetwork(":7895", nil)
+	require.NoError(t, err)
+	historyNetwork2, err := genHistoryNetwork(":7896", []*enode.Node{historyNetwork1.portalProtocol.Self()})
+	require.NoError(t, err)
+	// wait node start
+	time.Sleep(10 * time.Second)
+
+	entryMap, err := parseDataForBlock14764013()
+	require.NoError(t, err)
+
+	headerEntry := entryMap["header"]
+
+	// test GetBlockHeader
+	// no content
+	header, err := historyNetwork2.GetBlockHeader(headerEntry.key[1:])
+	require.Error(t, err)
+	require.Nil(t, header)
+
+	contentId := historyNetwork1.portalProtocol.ToContentId(headerEntry.key)
+	err = historyNetwork1.portalProtocol.Put(contentId, headerEntry.value)
+	require.NoError(t, err)
+	// get content from historyNetwork1
+	header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
+	require.NoError(t, err)
+	require.NotNil(t, header)
+	// get content from local
+	header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
+	require.NoError(t, err)
+	require.NotNil(t, header)
+}
+
+type contentEntry struct {
+	key   []byte
+	value []byte
+}
+
+func parseBlockHeaderKeyContent() ([]contentEntry, error) {
+	headWithProofBytes, err := os.ReadFile("./testdata/header_with_proofs.json")
+	if err != nil {
+		return nil, err
+	}
+	headerMap := make(map[string]map[string]string)
+
+	err = json.Unmarshal(headWithProofBytes, &headerMap)
+	if err != nil {
+		return nil, err
+	}
+	res := make([]contentEntry, 0)
+	for _, v := range headerMap {
+		entry := contentEntry{}
+		val := v["value"]
+		bytes, err := hexutil.Decode(val)
+		if err != nil {
+			return nil, err
+		}
+		entry.value = bytes
+		key := v["content_key"]
+		keyBytes, err := hexutil.Decode(key)
+		if err != nil {
+			return nil, err
+		}
+		entry.key = keyBytes
+		res = append(res, entry)
+	}
+	return res, nil
+}
+
+type MockStorage struct {
+	db map[string][]byte
+}
+
+func (m *MockStorage) Get(contentId []byte) ([]byte, error) {
+	if content, ok := m.db[string(contentId)]; ok {
+		return content, nil
+	}
+	return nil, storage.ErrContentNotFound
+}
+
+func (m *MockStorage) Put(contentId []byte, content []byte) error {
+	m.db[string(contentId)] = content
+	return nil
+}
+
+func genHistoryNetwork(addr string, bootNodes []*enode.Node) (*HistoryNetwork, error) {
+	conf := discover.DefaultPortalProtocolConfig()
+	if addr != "" {
+		conf.ListenAddr = addr
+	}
+	if bootNodes != nil {
+		conf.BootstrapNodes = bootNodes
+	}
+
+	contentQueue := make(chan *discover.ContentElement, 50)
+
+	key, err := crypto.GenerateKey()
+	if err != nil {
+		panic("couldn't generate key: " + err.Error())
+	}
+
+	portalProtocol, err := discover.NewPortalProtocol(conf, portalwire.HistoryNetwork, key, &MockStorage{db: make(map[string][]byte)}, contentQueue)
+	if err != nil {
+		return nil, err
+	}
+
+	accu, err := NewMasterAccumulator()
+	if err != nil {
+		return nil, err
+	}
+
+	err = portalProtocol.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewHistoryNetwork(portalProtocol, &accu), nil
+}
+
+func parseDataForBlock14764013() (map[string]contentEntry, error) {
+	content, err := os.ReadFile("./testdata/block_14764013.json")
+	if err != nil {
+		return nil, err
+	}
+
+	contentMap := make(map[string]map[string]string)
+	json.Unmarshal(content, &contentMap)
+	res := make(map[string]contentEntry)
+	for key, val := range contentMap {
+		entry := contentEntry{}
+		contentKey := val["content_key"]
+		entry.key, err = hexutil.Decode(contentKey)
+		if err != nil {
+			return nil, err
+		}
+		entry.value, err = hexutil.Decode(val["content_value"])
+		if err != nil {
+			return nil, err
+		}
+		res[key] = entry
+	}
+	return res, nil
+}