feat: new history key and content

portalnetwork/history/history_network.go

@@ -5,8 +5,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/protolambda/ztyp/codec"
+	"github.com/protolambda/ztyp/view"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -19,10 +22,11 @@ import (
 type ContentType byte
 
 const (
-	BlockHeaderType      ContentType = 0x00
+	BlockHeaderType       ContentType = 0x00
-	BlockBodyType        ContentType = 0x01
+	BlockBodyType         ContentType = 0x01
-	ReceiptsType         ContentType = 0x02
+	ReceiptsType          ContentType = 0x02
-	EpochAccumulatorType ContentType = 0x03
+	BlockHeaderNumberType ContentType = 0x03
+	// EpochAccumulatorType ContentType = 0x03
 )
 
 var (
@@ -33,6 +37,7 @@ var (
 	ErrContentOutOfRange        = errors.New("content out of range")
 	ErrHeaderWithProofIsInvalid = errors.New("header proof is invalid")
 	ErrInvalidBlockHash         = errors.New("invalid block hash")
+	ErrInvalidBlockNumber       = errors.New("invalid block number")
 )
 
 var emptyReceiptHash = hexutil.MustDecode("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
@@ -250,49 +255,49 @@ func (h *HistoryNetwork) GetReceipts(blockHash []byte) ([]*types.Receipt, error)
 	return nil, storage.ErrContentNotFound
 }
 
-func (h *HistoryNetwork) GetEpochAccumulator(epochHash []byte) (*EpochAccumulator, error) {
+// func (h *HistoryNetwork) GetEpochAccumulator(epochHash []byte) (*EpochAccumulator, error) {
-	contentKey := newContentKey(EpochAccumulatorType, epochHash).encode()
+// 	contentKey := newContentKey(EpochAccumulatorType, epochHash).encode()
-	contentId := h.portalProtocol.ToContentId(contentKey)
+// 	contentId := h.portalProtocol.ToContentId(contentKey)
 
-	res, err := h.portalProtocol.Get(contentKey, contentId)
+// 	res, err := h.portalProtocol.Get(contentKey, contentId)
-	// other error
+// 	// other error
-	if err != nil && !errors.Is(err, storage.ErrContentNotFound) {
+// 	if err != nil && !errors.Is(err, storage.ErrContentNotFound) {
-		return nil, err
+// 		return nil, err
-	}
+// 	}
-	// no error
+// 	// no error
-	if err == nil {
+// 	if err == nil {
-		epochAccu, err := decodeEpochAccumulator(res)
+// 		epochAccu, err := decodeEpochAccumulator(res)
-		return epochAccu, err
+// 		return epochAccu, err
-	}
+// 	}
-	for retries := 0; retries < requestRetries; retries++ {
+// 	for retries := 0; retries < requestRetries; retries++ {
-		content, _, err := h.portalProtocol.ContentLookup(contentKey, contentId)
+// 		content, _, err := h.portalProtocol.ContentLookup(contentKey, contentId)
-		if err != nil {
+// 		if err != nil {
-			h.log.Error("getEpochAccumulator failed", "contentKey", hexutil.Encode(contentKey), "err", err)
+// 			h.log.Error("getEpochAccumulator failed", "contentKey", hexutil.Encode(contentKey), "err", err)
-			continue
+// 			continue
-		}
+// 		}
-		epochAccu, err := decodeEpochAccumulator(content)
+// 		epochAccu, err := decodeEpochAccumulator(content)
-		if err != nil {
+// 		if err != nil {
-			h.log.Error("decodeEpochAccumulator failed", "content", hexutil.Encode(content), "err", err)
+// 			h.log.Error("decodeEpochAccumulator failed", "content", hexutil.Encode(content), "err", err)
-			continue
+// 			continue
-		}
+// 		}
-		hash, err := epochAccu.HashTreeRoot()
+// 		hash, err := epochAccu.HashTreeRoot()
-		if err != nil {
+// 		if err != nil {
-			h.log.Error("hashTreeRoot failed", "err", err)
+// 			h.log.Error("hashTreeRoot failed", "err", err)
-			continue
+// 			continue
-		}
+// 		}
-		mixHash := MixInLength(hash, epochSize)
+// 		mixHash := MixInLength(hash, epochSize)
-		if !bytes.Equal(mixHash, epochHash) {
+// 		if !bytes.Equal(mixHash, epochHash) {
-			h.log.Error("epochHash is not equal", "mixHash", hexutil.Encode(mixHash), "epochHash", hexutil.Encode(epochHash))
+// 			h.log.Error("epochHash is not equal", "mixHash", hexutil.Encode(mixHash), "epochHash", hexutil.Encode(epochHash))
-			continue
+// 			continue
-		}
+// 		}
-		err = h.portalProtocol.Put(contentKey, contentId, content)
+// 		err = h.portalProtocol.Put(contentKey, contentId, content)
-		if err != nil {
+// 		if err != nil {
-			h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
+// 			h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
-		}
+// 		}
-		return epochAccu, nil
+// 		return epochAccu, nil
-	}
+// 	}
-	return nil, storage.ErrContentNotFound
+// 	return nil, storage.ErrContentNotFound
-}
+// }
 
 func (h *HistoryNetwork) verifyHeader(header *types.Header, proof BlockHeaderProof) (bool, error) {
 	return h.masterAccumulator.VerifyHeader(*header, proof)
@@ -534,10 +539,13 @@ func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) erro
 		if err != nil {
 			return err
 		}
-		header, err := ValidateBlockHeaderBytes(headerWithProof.Header, contentKey[1:])
+		header, err := DecodeBlockHeader(headerWithProof.Header)
 		if err != nil {
 			return err
 		}
+		if !bytes.Equal(header.Hash().Bytes(), contentKey[1:]) {
+			return ErrInvalidBlockHash
+		}
 		valid, err := h.verifyHeader(header, *headerWithProof.Proof)
 		if err != nil {
 			return err
@@ -566,25 +574,31 @@ func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) erro
 		}
 		_, err = ValidatePortalReceiptsBytes(content, header.ReceiptHash.Bytes())
 		return err
-	case EpochAccumulatorType:
+	case BlockHeaderNumberType:
-		if !h.masterAccumulator.Contains(contentKey[1:]) {
+		headerWithProof, err := DecodeBlockHeaderWithProof(content)
-			return errors.New("epoch hash is not existed")
-		}
-
-		epochAcc, err := decodeEpochAccumulator(content)
 		if err != nil {
 			return err
 		}
-		hash, err := epochAcc.HashTreeRoot()
+		header, err := DecodeBlockHeader(headerWithProof.Header)
 		if err != nil {
 			return err
 		}
-
+		blockNumber := view.Uint64View(0)
-		epochHash := MixInLength(hash, epochSize)
+		err = blockNumber.Deserialize(codec.NewDecodingReader(bytes.NewReader(contentKey[1:]), uint64(len(contentKey[1:]))))
-		if !bytes.Equal(contentKey[1:], epochHash) {
+		if err != nil {
-			return errors.New("epoch accumulator has invalid root hash")
+			return err
 		}
-		return nil
+		if header.Number.Cmp(big.NewInt(int64(blockNumber))) != 0 {
+			return ErrInvalidBlockNumber
+		}
+		valid, err := h.verifyHeader(header, *headerWithProof.Proof)
+		if err != nil {
+			return err
+		}
+		if !valid {
+			return ErrHeaderWithProofIsInvalid
+		}
+		return err
 	}
 	return errors.New("unknown content type")
 }
@@ -616,6 +630,15 @@ func ValidateBlockHeaderBytes(headerBytes []byte, blockHash []byte) (*types.Head
 	return header, nil
 }
 
+func DecodeBlockHeader(headerBytes []byte) (*types.Header, error) {
+	header := new(types.Header)
+	err := rlp.DecodeBytes(headerBytes, header)
+	if err != nil {
+		return nil, err
+	}
+	return header, nil
+}
+
 func DecodeBlockHeaderWithProof(content []byte) (*BlockHeaderWithProof, error) {
 	headerWithProof := new(BlockHeaderWithProof)
 	err := headerWithProof.UnmarshalSSZ(content)

portalnetwork/history/history_network_test.go

@@ -4,9 +4,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	_ "embed"
-	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"math/big"
 	"net"
 	"os"
@@ -20,7 +18,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover/portalwire"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/portalnetwork/storage"
-	"github.com/holiman/uint256"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
@@ -36,70 +33,6 @@ func ContentId(contentKey []byte) []byte {
 	return digest[:]
 }
 
-// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
-func TestContentKey(t *testing.T) {
-	testCases := []struct {
-		name          string
-		hash          string
-		contentKey    string
-		contentIdHex  string
-		contentIdU256 string
-		selector      ContentType
-	}{
-		{
-			name:          "block header key",
-			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentKey:    "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentIdHex:  "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
-			contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
-			selector:      BlockHeaderType,
-		},
-		{
-			name:          "block body key",
-			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentKey:    "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentIdHex:  "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
-			contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
-			selector:      BlockBodyType,
-		},
-		{
-			name:          "receipt key",
-			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentKey:    "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
-			contentIdHex:  "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
-			contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
-			selector:      ReceiptsType,
-		},
-		{
-			name:          "epoch accumelator key",
-			hash:          "e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
-			contentKey:    "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
-			contentIdHex:  "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e",
-			contentIdU256: "72232402989179419196382321898161638871438419016077939952896528930608027961710",
-			selector:      EpochAccumulatorType,
-		},
-	}
-
-	for _, c := range testCases {
-		t.Run(c.name, func(t *testing.T) {
-			hashByte, err := hex.DecodeString(c.hash)
-			require.NoError(t, err)
-
-			contentKey := newContentKey(c.selector, hashByte).encode()
-			hexKey := hex.EncodeToString(contentKey)
-			require.Equal(t, hexKey, c.contentKey)
-			contentId := ContentId(contentKey)
-			require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
-
-			bigNum := big.NewInt(0).SetBytes(contentId)
-			u256Format, isOverflow := uint256.FromBig(bigNum)
-			require.False(t, isOverflow)
-			u256Str := fmt.Sprint(u256Format)
-			require.Equal(t, u256Str, c.contentIdU256)
-		})
-	}
-}
-
 func TestValidateHeader(t *testing.T) {
 	entrys, err := parseBlockHeaderKeyContent()
 	require.NoError(t, err)
@@ -182,16 +115,7 @@ func TestValidateEpochAccu(t *testing.T) {
 	require.NoError(t, err)
 	root := MixInLength(epochRoot, epochSize)
 
-	err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, root).encode(), epochAccuBytes)
+	require.True(t, historyNetwork.masterAccumulator.Contains(root))
-	require.NoError(t, err)
-
-	// invalid root hash
-	err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, epochRoot[:]).encode(), epochAccuBytes)
-	require.Error(t, err)
-	// invalid epoch data
-	epochAccuBytes[len(epochAccuBytes)-1] = 0xaa
-	err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, root).encode(), epochAccuBytes)
-	require.Error(t, err)
 }
 
 func TestGetContentByKey(t *testing.T) {
@@ -267,33 +191,25 @@ func TestGetContentByKey(t *testing.T) {
 		return
 	}
 
-	// test GetEpoch
+	headerNumberEntry := entryMap["headerBlock"]
-	epochAccuBytes, err := hexutil.Decode(epochAccuHex)
-	require.NoError(t, err)
-	epochAccu, err := decodeEpochAccumulator(epochAccuBytes)
-	require.NoError(t, err)
-	epochRoot, err := epochAccu.HashTreeRoot()
-	require.NoError(t, err)
-	root := MixInLength(epochRoot, epochSize)
 
-	contentKey := newContentKey(EpochAccumulatorType, root).encode()
+	// test GetBlockHeader
-	content := epochAccuBytes
+	// no content
-
+	header, err = historyNetwork2.GetBlockHeader(headerNumberEntry.key[1:])
-	epoch, err := historyNetwork2.GetEpochAccumulator(contentKey[1:])
 	require.Error(t, err)
-	require.Nil(t, epoch)
+	require.Nil(t, header)
 
-	contentId = historyNetwork1.portalProtocol.ToContentId(contentKey)
+	contentId = historyNetwork1.portalProtocol.ToContentId(headerNumberEntry.key)
-	err = historyNetwork1.portalProtocol.Put(contentKey, contentId, content)
+	err = historyNetwork1.portalProtocol.Put(headerEntry.key, contentId, headerEntry.value)
 	require.NoError(t, err)
 	// get content from historyNetwork1
-	epoch, err = historyNetwork2.GetEpochAccumulator(contentKey[1:])
+	header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
 	require.NoError(t, err)
-	require.NotNil(t, epoch)
+	require.NotNil(t, header)
 	// get content from local
-	epoch, err = historyNetwork2.GetEpochAccumulator(contentKey[1:])
+	header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
 	require.NoError(t, err)
-	require.NotNil(t, epoch)
+	require.NotNil(t, header)
 }
 
 type Entry struct {

portalnetwork/history/types_test.go

@@ -0,0 +1,95 @@
+package history
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/holiman/uint256"
+	"github.com/protolambda/ztyp/codec"
+	"github.com/protolambda/ztyp/view"
+	"github.com/stretchr/testify/require"
+)
+
+// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
+func TestContentKey(t *testing.T) {
+	testCases := []struct {
+		name          string
+		hash          string
+		contentKey    string
+		contentIdHex  string
+		contentIdU256 string
+		selector      ContentType
+	}{
+		{
+			name:          "block header key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
+			contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
+			selector:      BlockHeaderType,
+		},
+		{
+			name:          "block body key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
+			contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
+			selector:      BlockBodyType,
+		},
+		{
+			name:          "receipt key",
+			hash:          "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentKey:    "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
+			contentIdHex:  "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
+			contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
+			selector:      ReceiptsType,
+		},
+	}
+
+	for _, c := range testCases {
+		t.Run(c.name, func(t *testing.T) {
+			hashByte, err := hex.DecodeString(c.hash)
+			require.NoError(t, err)
+
+			contentKey := newContentKey(c.selector, hashByte).encode()
+			hexKey := hex.EncodeToString(contentKey)
+			require.Equal(t, hexKey, c.contentKey)
+			contentId := ContentId(contentKey)
+			require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
+
+			bigNum := big.NewInt(0).SetBytes(contentId)
+			u256Format, isOverflow := uint256.FromBig(bigNum)
+			require.False(t, isOverflow)
+			u256Str := fmt.Sprint(u256Format)
+			require.Equal(t, u256Str, c.contentIdU256)
+		})
+	}
+}
+
+func TestBlockNumber(t *testing.T) {
+	blockNumber := 12345678
+	contentKey := "0x034e61bc0000000000"
+	contentId := "0x2113990747a85ab39785d21342fa5db1f68acc0011605c0c73f68fc331643dcf"
+	contentIdU256 := "14960950260935695396511307566164035182676768442501235074589175304147024756175"
+
+	key := view.Uint64View(blockNumber)
+	var buf bytes.Buffer
+	err := key.Serialize(codec.NewEncodingWriter(&buf))
+	require.NoError(t, err)
+	keyData := []byte{byte(BlockHeaderNumberType)}
+	keyData = append(keyData, buf.Bytes()...)
+	require.Equal(t, hexutil.MustDecode(contentKey), keyData)
+
+	contentIdData := ContentId(keyData)
+	require.Equal(t, contentId, hexutil.Encode(contentIdData))
+
+	bigNum := big.NewInt(0).SetBytes(contentIdData)
+	u256Format, isOverflow := uint256.FromBig(bigNum)
+	require.False(t, isOverflow)
+	u256Str := fmt.Sprint(u256Format)
+	require.Equal(t, u256Str, contentIdU256)
+}

portalnetwork/history/types_zrnt.go

@@ -4,6 +4,7 @@ import (
 	"github.com/protolambda/zrnt/eth2/beacon/common"
 	"github.com/protolambda/ztyp/codec"
 	"github.com/protolambda/ztyp/tree"
+	"github.com/protolambda/ztyp/view"
 )
 
 const beaconBlockBodyProofLen = 8
@@ -193,3 +194,5 @@ func (h HistoricalRoots) HashTreeRoot(spec *common.Spec, hFn tree.HashFn) common
 		return nil
 	}, length, uint64(spec.HISTORICAL_ROOTS_LIMIT))
 }
+
+type BlockNumberKey view.Uint64View