forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 06:36:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: new history key and content

Browse source
This commit is contained in:
fearlseefe 2024-09-24 15:21:33 +08:00 committed by Chen Kai
parent 7c5231ce74
commit 0fa168fd81
5 changed files with 209 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
portalnetwork/history/history_network.go
View file

@ -5,8 +5,11 @@ import (
"context"
"errors"
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/protolambda/ztyp/codec"
"github.com/protolambda/ztyp/view"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
@ -19,10 +22,11 @@ import (
type ContentType byte
const (
BlockHeaderType ContentType = 0x00
BlockBodyType ContentType = 0x01
ReceiptsType ContentType = 0x02
EpochAccumulatorType ContentType = 0x03
BlockHeaderType ContentType = 0x00
BlockBodyType ContentType = 0x01
ReceiptsType ContentType = 0x02
BlockHeaderNumberType ContentType = 0x03
// EpochAccumulatorType ContentType = 0x03
)
var (
@ -33,6 +37,7 @@ var (
ErrContentOutOfRange = errors.New("content out of range")
ErrHeaderWithProofIsInvalid = errors.New("header proof is invalid")
ErrInvalidBlockHash = errors.New("invalid block hash")
ErrInvalidBlockNumber = errors.New("invalid block number")
)
var emptyReceiptHash = hexutil.MustDecode("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
@ -250,49 +255,49 @@ func (h *HistoryNetwork) GetReceipts(blockHash []byte) ([]*types.Receipt, error)
return nil, storage.ErrContentNotFound
}
func (h *HistoryNetwork) GetEpochAccumulator(epochHash []byte) (*EpochAccumulator, error) {
contentKey := newContentKey(EpochAccumulatorType, epochHash).encode()
contentId := h.portalProtocol.ToContentId(contentKey)
// func (h *HistoryNetwork) GetEpochAccumulator(epochHash []byte) (*EpochAccumulator, error) {
// contentKey := newContentKey(EpochAccumulatorType, epochHash).encode()
// contentId := h.portalProtocol.ToContentId(contentKey)
res, err := h.portalProtocol.Get(contentKey, contentId)
// other error
if err != nil && !errors.Is(err, storage.ErrContentNotFound) {
return nil, err
}
// no error
if err == nil {
epochAccu, err := decodeEpochAccumulator(res)
return epochAccu, err
}
for retries := 0; retries < requestRetries; retries++ {
content, _, err := h.portalProtocol.ContentLookup(contentKey, contentId)
if err != nil {
h.log.Error("getEpochAccumulator failed", "contentKey", hexutil.Encode(contentKey), "err", err)
continue
}
epochAccu, err := decodeEpochAccumulator(content)
if err != nil {
h.log.Error("decodeEpochAccumulator failed", "content", hexutil.Encode(content), "err", err)
continue
}
hash, err := epochAccu.HashTreeRoot()
if err != nil {
h.log.Error("hashTreeRoot failed", "err", err)
continue
}
mixHash := MixInLength(hash, epochSize)
if !bytes.Equal(mixHash, epochHash) {
h.log.Error("epochHash is not equal", "mixHash", hexutil.Encode(mixHash), "epochHash", hexutil.Encode(epochHash))
continue
}
err = h.portalProtocol.Put(contentKey, contentId, content)
if err != nil {
h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
}
return epochAccu, nil
}
return nil, storage.ErrContentNotFound
}
// res, err := h.portalProtocol.Get(contentKey, contentId)
// // other error
// if err != nil && !errors.Is(err, storage.ErrContentNotFound) {
// return nil, err
// }
// // no error
// if err == nil {
// epochAccu, err := decodeEpochAccumulator(res)
// return epochAccu, err
// }
// for retries := 0; retries < requestRetries; retries++ {
// content, _, err := h.portalProtocol.ContentLookup(contentKey, contentId)
// if err != nil {
// h.log.Error("getEpochAccumulator failed", "contentKey", hexutil.Encode(contentKey), "err", err)
// continue
// }
// epochAccu, err := decodeEpochAccumulator(content)
// if err != nil {
// h.log.Error("decodeEpochAccumulator failed", "content", hexutil.Encode(content), "err", err)
// continue
// }
// hash, err := epochAccu.HashTreeRoot()
// if err != nil {
// h.log.Error("hashTreeRoot failed", "err", err)
// continue
// }
// mixHash := MixInLength(hash, epochSize)
// if !bytes.Equal(mixHash, epochHash) {
// h.log.Error("epochHash is not equal", "mixHash", hexutil.Encode(mixHash), "epochHash", hexutil.Encode(epochHash))
// continue
// }
// err = h.portalProtocol.Put(contentKey, contentId, content)
// if err != nil {
// h.log.Error("failed to store content in getReceipts", "contentKey", hexutil.Encode(contentKey), "content", hexutil.Encode(content))
// }
// return epochAccu, nil
// }
// return nil, storage.ErrContentNotFound
// }
func (h *HistoryNetwork) verifyHeader(header *types.Header, proof BlockHeaderProof) (bool, error) {
return h.masterAccumulator.VerifyHeader(*header, proof)
@ -534,10 +539,13 @@ func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) erro
if err != nil {
return err
}
header, err := ValidateBlockHeaderBytes(headerWithProof.Header, contentKey[1:])
header, err := DecodeBlockHeader(headerWithProof.Header)
if err != nil {
return err
}
if !bytes.Equal(header.Hash().Bytes(), contentKey[1:]) {
return ErrInvalidBlockHash
}
valid, err := h.verifyHeader(header, *headerWithProof.Proof)
if err != nil {
return err
@ -566,25 +574,31 @@ func (h *HistoryNetwork) validateContent(contentKey []byte, content []byte) erro
}
_, err = ValidatePortalReceiptsBytes(content, header.ReceiptHash.Bytes())
return err
case EpochAccumulatorType:
if !h.masterAccumulator.Contains(contentKey[1:]) {
return errors.New("epoch hash is not existed")
}
epochAcc, err := decodeEpochAccumulator(content)
case BlockHeaderNumberType:
headerWithProof, err := DecodeBlockHeaderWithProof(content)
if err != nil {
return err
}
hash, err := epochAcc.HashTreeRoot()
header, err := DecodeBlockHeader(headerWithProof.Header)
if err != nil {
return err
}
epochHash := MixInLength(hash, epochSize)
if !bytes.Equal(contentKey[1:], epochHash) {
return errors.New("epoch accumulator has invalid root hash")
blockNumber := view.Uint64View(0)
err = blockNumber.Deserialize(codec.NewDecodingReader(bytes.NewReader(contentKey[1:]), uint64(len(contentKey[1:]))))
if err != nil {
return err
}
return nil
if header.Number.Cmp(big.NewInt(int64(blockNumber))) != 0 {
return ErrInvalidBlockNumber
}
valid, err := h.verifyHeader(header, *headerWithProof.Proof)
if err != nil {
return err
}
if !valid {
return ErrHeaderWithProofIsInvalid
}
return err
}
return errors.New("unknown content type")
}
@ -616,6 +630,15 @@ func ValidateBlockHeaderBytes(headerBytes []byte, blockHash []byte) (*types.Head
return header, nil
}
func DecodeBlockHeader(headerBytes []byte) (*types.Header, error) {
header := new(types.Header)
err := rlp.DecodeBytes(headerBytes, header)
if err != nil {
return nil, err
}
return header, nil
}
func DecodeBlockHeaderWithProof(content []byte) (*BlockHeaderWithProof, error) {
headerWithProof := new(BlockHeaderWithProof)
err := headerWithProof.UnmarshalSSZ(content)

108
portalnetwork/history/history_network_test.go
View file

@ -4,9 +4,7 @@ import (
"bytes"
"crypto/sha256"
_ "embed"
"encoding/hex"
"encoding/json"
"fmt"
"math/big"
"net"
"os"
@ -20,7 +18,6 @@ import (
"github.com/ethereum/go-ethereum/p2p/discover/portalwire"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
@ -36,70 +33,6 @@ func ContentId(contentKey []byte) []byte {
return digest[:]
}
// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
func TestContentKey(t *testing.T) {
testCases := []struct {
name string
hash string
contentKey string
contentIdHex string
contentIdU256 string
selector ContentType
}{
{
name: "block header key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
selector: BlockHeaderType,
},
{
name: "block body key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
selector: BlockBodyType,
},
{
name: "receipt key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
selector: ReceiptsType,
},
{
name: "epoch accumelator key",
hash: "e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
contentKey: "03e242814b90ed3950e13aac7e56ce116540c71b41d1516605aada26c6c07cc491",
contentIdHex: "9fb2175e76c6989e0fdac3ee10c40d2a81eb176af32e1c16193e3904fe56896e",
contentIdU256: "72232402989179419196382321898161638871438419016077939952896528930608027961710",
selector: EpochAccumulatorType,
},
}
for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
hashByte, err := hex.DecodeString(c.hash)
require.NoError(t, err)
contentKey := newContentKey(c.selector, hashByte).encode()
hexKey := hex.EncodeToString(contentKey)
require.Equal(t, hexKey, c.contentKey)
contentId := ContentId(contentKey)
require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
bigNum := big.NewInt(0).SetBytes(contentId)
u256Format, isOverflow := uint256.FromBig(bigNum)
require.False(t, isOverflow)
u256Str := fmt.Sprint(u256Format)
require.Equal(t, u256Str, c.contentIdU256)
})
}
}
func TestValidateHeader(t *testing.T) {
entrys, err := parseBlockHeaderKeyContent()
require.NoError(t, err)
@ -182,16 +115,7 @@ func TestValidateEpochAccu(t *testing.T) {
require.NoError(t, err)
root := MixInLength(epochRoot, epochSize)
err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, root).encode(), epochAccuBytes)
require.NoError(t, err)
// invalid root hash
err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, epochRoot[:]).encode(), epochAccuBytes)
require.Error(t, err)
// invalid epoch data
epochAccuBytes[len(epochAccuBytes)-1] = 0xaa
err = historyNetwork.validateContent(newContentKey(EpochAccumulatorType, root).encode(), epochAccuBytes)
require.Error(t, err)
require.True(t, historyNetwork.masterAccumulator.Contains(root))
}
func TestGetContentByKey(t *testing.T) {
@ -267,33 +191,25 @@ func TestGetContentByKey(t *testing.T) {
return
}
// test GetEpoch
epochAccuBytes, err := hexutil.Decode(epochAccuHex)
require.NoError(t, err)
epochAccu, err := decodeEpochAccumulator(epochAccuBytes)
require.NoError(t, err)
epochRoot, err := epochAccu.HashTreeRoot()
require.NoError(t, err)
root := MixInLength(epochRoot, epochSize)
headerNumberEntry := entryMap["headerBlock"]
contentKey := newContentKey(EpochAccumulatorType, root).encode()
content := epochAccuBytes
epoch, err := historyNetwork2.GetEpochAccumulator(contentKey[1:])
// test GetBlockHeader
// no content
header, err = historyNetwork2.GetBlockHeader(headerNumberEntry.key[1:])
require.Error(t, err)
require.Nil(t, epoch)
require.Nil(t, header)
contentId = historyNetwork1.portalProtocol.ToContentId(contentKey)
err = historyNetwork1.portalProtocol.Put(contentKey, contentId, content)
contentId = historyNetwork1.portalProtocol.ToContentId(headerNumberEntry.key)
err = historyNetwork1.portalProtocol.Put(headerEntry.key, contentId, headerEntry.value)
require.NoError(t, err)
// get content from historyNetwork1
epoch, err = historyNetwork2.GetEpochAccumulator(contentKey[1:])
header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
require.NoError(t, err)
require.NotNil(t, epoch)
require.NotNil(t, header)
// get content from local
epoch, err = historyNetwork2.GetEpochAccumulator(contentKey[1:])
header, err = historyNetwork2.GetBlockHeader(headerEntry.key[1:])
require.NoError(t, err)
require.NotNil(t, epoch)
require.NotNil(t, header)
}
type Entry struct {

30
portalnetwork/history/testdata/block_14764013.json vendored
View file

File diff suppressed because one or more lines are too long

95
portalnetwork/history/types_test.go Normal file
View file

@ -0,0 +1,95 @@
package history
import (
"bytes"
"encoding/hex"
"fmt"
"math/big"
"testing"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/holiman/uint256"
"github.com/protolambda/ztyp/codec"
"github.com/protolambda/ztyp/view"
"github.com/stretchr/testify/require"
)
// testcases from https://github.com/ethereum/portal-network-specs/blob/master/content-keys-test-vectors.md
func TestContentKey(t *testing.T) {
testCases := []struct {
name string
hash string
contentKey string
contentIdHex string
contentIdU256 string
selector ContentType
}{
{
name: "block header key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "00d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "3e86b3767b57402ea72e369ae0496ce47cc15be685bec3b4726b9f316e3895fe",
contentIdU256: "28281392725701906550238743427348001871342819822834514257505083923073246729726",
selector: BlockHeaderType,
},
{
name: "block body key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "01d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "ebe414854629d60c58ddd5bf60fd72e41760a5f7a463fdcb169f13ee4a26786b",
contentIdU256: "106696502175825986237944249828698290888857178633945273402044845898673345165419",
selector: BlockBodyType,
},
{
name: "receipt key",
hash: "d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentKey: "02d1c390624d3bd4e409a61a858e5dcc5517729a9170d014a6c96530d64dd8621d",
contentIdHex: "a888f4aafe9109d495ac4d4774a6277c1ada42035e3da5e10a04cc93247c04a4",
contentIdU256: "76230538398907151249589044529104962263309222250374376758768131420767496438948",
selector: ReceiptsType,
},
}
for _, c := range testCases {
t.Run(c.name, func(t *testing.T) {
hashByte, err := hex.DecodeString(c.hash)
require.NoError(t, err)
contentKey := newContentKey(c.selector, hashByte).encode()
hexKey := hex.EncodeToString(contentKey)
require.Equal(t, hexKey, c.contentKey)
contentId := ContentId(contentKey)
require.Equal(t, c.contentIdHex, hex.EncodeToString(contentId))
bigNum := big.NewInt(0).SetBytes(contentId)
u256Format, isOverflow := uint256.FromBig(bigNum)
require.False(t, isOverflow)
u256Str := fmt.Sprint(u256Format)
require.Equal(t, u256Str, c.contentIdU256)
})
}
}
func TestBlockNumber(t *testing.T) {
blockNumber := 12345678
contentKey := "0x034e61bc0000000000"
contentId := "0x2113990747a85ab39785d21342fa5db1f68acc0011605c0c73f68fc331643dcf"
contentIdU256 := "14960950260935695396511307566164035182676768442501235074589175304147024756175"
key := view.Uint64View(blockNumber)
var buf bytes.Buffer
err := key.Serialize(codec.NewEncodingWriter(&buf))
require.NoError(t, err)
keyData := []byte{byte(BlockHeaderNumberType)}
keyData = append(keyData, buf.Bytes()...)
require.Equal(t, hexutil.MustDecode(contentKey), keyData)
contentIdData := ContentId(keyData)
require.Equal(t, contentId, hexutil.Encode(contentIdData))
bigNum := big.NewInt(0).SetBytes(contentIdData)
u256Format, isOverflow := uint256.FromBig(bigNum)
require.False(t, isOverflow)
u256Str := fmt.Sprint(u256Format)
require.Equal(t, u256Str, contentIdU256)
}

3
portalnetwork/history/types_zrnt.go
View file

@ -4,6 +4,7 @@ import (
"github.com/protolambda/zrnt/eth2/beacon/common"
"github.com/protolambda/ztyp/codec"
"github.com/protolambda/ztyp/tree"
"github.com/protolambda/ztyp/view"
)
const beaconBlockBodyProofLen = 8
@ -193,3 +194,5 @@ func (h HistoricalRoots) HashTreeRoot(spec *common.Spec, hFn tree.HashFn) common
return nil
}, length, uint64(spec.HISTORICAL_ROOTS_LIMIT))
}
type BlockNumberKey view.Uint64View