forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-06 12:03:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: beacon-storage-part1

Browse source 
impl get and put for LightClientBootstrap and LightClientUpdatesByRange
This commit is contained in:
fearlessfe 2024-03-25 21:24:33 +08:00 committed by Chen Kai
parent f7834b5be1
commit 74e0424686
9 changed files with 359 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
portalnetwork/beacon/sql.go Normal file
View file

@ -0,0 +1,45 @@
package beacon
const CreateQueryDBBeacon = `CREATE TABLE IF NOT EXISTS beacon (
content_id blob PRIMARY KEY,
content_key blob NOT NULL,
content_value blob NOT NULL,
content_size INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS beacon_content_size_idx ON beacon(content_size);`
const InsertQueryBeacon = `INSERT OR IGNORE INTO beacon (content_id, content_key, content_value, content_size)
VALUES (?1, ?2, ?3, ?4)`
const DeleteQueryBeacon = `DELETE FROM beacon
WHERE content_id = (?1)`
const ContentKeyLookupQueryBeacon = `SELECT content_key FROM beacon WHERE content_id = (?1) LIMIT 1`
const ContentValueLookupQueryBeacon = `SELECT content_value FROM beacon WHERE content_id = (?1) LIMIT 1`
const TotalDataSizeQueryBeacon = "SELECT TOTAL(content_size) FROM beacon"
const TotalEntryCountQueryBeacon = "SELECT COUNT(*) FROM beacon"
const ContentSizeLookupQueryBeacon = "SELECT content_size FROM beacon WHERE content_id = (?1)"
const LCUpdateCreateTable = `CREATE TABLE IF NOT EXISTS lc_update (
period INTEGER PRIMARY KEY,
value BLOB NOT NULL,
score INTEGER NOT NULL,
update_size INTEGER
);
CREATE INDEX IF NOT EXISTS update_size_idx ON lc_update(update_size);
DROP INDEX IF EXISTS period_idx;`
const InsertLCUpdateQuery = `INSERT OR IGNORE INTO lc_update (period, value, score, update_size)
VALUES (?1, ?2, ?3, ?4)`
const LCUpdateLookupQuery = `SELECT value FROM lc_update WHERE period = (?1) LIMIT 1`
const LCUpdateLookupQueryByRange = `SELECT value FROM lc_update WHERE period >= (?1) AND period < (?2)`
const LCUpdatePeriodLookupQuery = `SELECT period FROM lc_update WHERE period = (?1) LIMIT 1`
const LCUpdateTotalSizeQuery = `SELECT TOTAL(update_size) FROM lc_update`

152
portalnetwork/beacon/storage.go Normal file
View file

@ -0,0 +1,152 @@
package beacon
import (
"bytes"
"context"
"database/sql"
"github.com/ethereum/go-ethereum/log"
"github.com/protolambda/zrnt/eth2/beacon/common"
"github.com/protolambda/ztyp/codec"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
)
const BytesInMB uint64 = 1000 * 1000
type BeaconStorage struct {
storageCapacityInBytes uint64
db *sql.DB
log log.Logger
spec *common.Spec
}
var _ storage.ContentStorage = &BeaconStorage{}
func NewBeaconStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) {
bs := &BeaconStorage{
storageCapacityInBytes: config.StorageCapacityMB * BytesInMB,
db: config.DB,
log: log.New("beacon_storage"),
spec: config.Spec,
}
if err := bs.setup(); err != nil {
return nil, err
}
return bs, nil
}
func (bs *BeaconStorage) setup() error {
if _, err := bs.db.Exec(CreateQueryDBBeacon); err != nil {
return err
}
if _, err := bs.db.Exec(LCUpdateCreateTable); err != nil {
return err
}
return nil
}
func (bs *BeaconStorage) Get(contentKey []byte, contentId []byte) ([]byte, error) {
switch storage.ContentType(contentKey[0]) {
case LightClientBootstrap:
return bs.getContentValue(contentId)
case LightClientUpdate:
lightClientUpdateKey := new(LightClientUpdateKey)
err := lightClientUpdateKey.UnmarshalSSZ(contentKey[1:])
if err != nil {
return nil, err
}
return bs.getLcUpdateValueByRange(lightClientUpdateKey.StartPeriod, lightClientUpdateKey.StartPeriod+lightClientUpdateKey.Count)
case LightClientFinalityUpdate:
case LightClientOptimisticUpdate:
}
return nil, nil
}
func (bs *BeaconStorage) Put(contentKey []byte, contentId []byte, content []byte) error {
switch storage.ContentType(contentKey[0]) {
case LightClientBootstrap:
return bs.putContentValue(contentId, contentKey, content)
case LightClientUpdate:
lightClientUpdateKey := new(LightClientUpdateKey)
err := lightClientUpdateKey.UnmarshalSSZ(contentKey[1:])
if err != nil {
return err
}
lightClientUpdateRange := new(LightClientUpdateRange)
reader := codec.NewDecodingReader(bytes.NewReader(content), uint64(len(content)))
err = lightClientUpdateRange.Deserialize(bs.spec, reader)
if err != nil {
return err
}
for index, update := range *lightClientUpdateRange {
var buf bytes.Buffer
writer := codec.NewEncodingWriter(&buf)
err := update.Serialize(bs.spec, writer)
if err != nil {
return err
}
period := lightClientUpdateKey.StartPeriod + uint64(index)
err = bs.putLcUpdate(period, buf.Bytes())
if err != nil {
return err
}
}
return nil
case LightClientFinalityUpdate:
case LightClientOptimisticUpdate:
}
return nil
}
func (bs *BeaconStorage) getContentValue(contentId []byte) ([]byte, error) {
res := make([]byte, 0)
err := bs.db.QueryRowContext(context.Background(), ContentValueLookupQueryBeacon, contentId).Scan(&res)
if err == sql.ErrNoRows {
return nil, storage.ErrContentNotFound
}
return res, err
}
func (bs *BeaconStorage) getLcUpdateValueByRange(start, end uint64) ([]byte, error) {
// LightClientUpdateRange := make([]ForkedLightClientUpdate, 0)
var lightClientUpdateRange LightClientUpdateRange
rows, err := bs.db.QueryContext(context.Background(), LCUpdateLookupQueryByRange, start, end)
if err != nil {
return nil, err
}
hasData := false
defer rows.Close()
for rows.Next() {
hasData = true
var val []byte
err := rows.Scan(&val)
if err != nil {
return nil, err
}
update := new(ForkedLightClientUpdate)
dec := codec.NewDecodingReader(bytes.NewReader(val), uint64(len(val)))
update.Deserialize(bs.spec, dec)
lightClientUpdateRange = append(lightClientUpdateRange, *update)
}
if !hasData {
return nil, storage.ErrContentNotFound
}
var buf bytes.Buffer
err = lightClientUpdateRange.Serialize(bs.spec, codec.NewEncodingWriter(&buf))
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (bs *BeaconStorage) putContentValue(contentId, contentKey, value []byte) error {
length := 32 + len(contentKey) + len(value)
_, err := bs.db.ExecContext(context.Background(), InsertQueryBeacon, contentId, contentKey, value, length)
return err
}
func (bs *BeaconStorage) putLcUpdate(period uint64, value []byte) error {
_, err := bs.db.ExecContext(context.Background(), InsertLCUpdateQuery, period, value, 0, len(value))
return err
}

119
portalnetwork/beacon/storage_test.go Normal file
View file

@ -0,0 +1,119 @@
package beacon
import (
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"os"
"path"
"testing"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/portalnetwork/storage"
"github.com/ethereum/go-ethereum/portalnetwork/utils"
"github.com/holiman/uint256"
_ "github.com/mattn/go-sqlite3"
"github.com/protolambda/zrnt/eth2/configs"
"github.com/stretchr/testify/require"
)
var zeroNodeId = uint256.NewInt(0).Bytes32()
const dbName = "beacon.sqlite"
func defaultContentIdFunc(contentKey []byte) []byte {
digest := sha256.Sum256(contentKey)
return digest[:]
}
func TestGetAndPut(t *testing.T) {
testDir := "./"
beaconStorage, err := genStorage(testDir)
require.NoError(t, err)
defer clearNodeData(testDir)
key, value, err := getClientBootstrap()
require.NoError(t, err)
contentId := defaultContentIdFunc(key)
_, err = beaconStorage.Get(key, contentId)
require.Equal(t, storage.ErrContentNotFound, err)
err = beaconStorage.Put(key, contentId, value)
require.NoError(t, err)
res, err := beaconStorage.Get(key, contentId)
require.NoError(t, err)
require.Equal(t, value, res)
key, value, err = getClientUpdatesByRange()
require.NoError(t, err)
contentId = defaultContentIdFunc(key)
_, err = beaconStorage.Get(key, contentId)
require.Equal(t, storage.ErrContentNotFound, err)
err = beaconStorage.Put(key, contentId, value)
require.NoError(t, err)
res, err = beaconStorage.Get(key, contentId)
require.NoError(t, err)
require.Equal(t, value, res)
}
func genStorage(testDir string) (storage.ContentStorage, error) {
utils.EnsureDir(testDir)
db, err := sql.Open("sqlite3", path.Join(testDir, dbName))
if err != nil {
return nil, err
}
config := &storage.PortalStorageConfig{
StorageCapacityMB: 1000,
DB: db,
NodeId: enode.ID(zeroNodeId),
Spec: configs.Mainnet,
}
return NewBeaconStorage(*config)
}
func getClientBootstrap() ([]byte, []byte, error) {
filePath := "testdata/light_client_bootstrap.json"
f, err := os.ReadFile(filePath)
if err != nil {
return nil, nil, err
}
var result map[string]map[string]string
err = json.Unmarshal(f, &result)
if err != nil {
return nil, nil, err
}
contentKey := hexutil.MustDecode(result["6718368"]["content_key"])
contentValue := hexutil.MustDecode(result["6718368"]["content_value"])
return contentKey, contentValue, nil
}
func getClientUpdatesByRange() ([]byte, []byte, error) {
filePath := "testdata/light_client_updates_by_range.json"
f, err := os.ReadFile(filePath)
if err != nil {
return nil, nil, err
}
var result map[string]map[string]string
err = json.Unmarshal(f, &result)
if err != nil {
return nil, nil, err
}
contentKey := hexutil.MustDecode(result["6684738"]["content_key"])
contentValue := hexutil.MustDecode(result["6684738"]["content_value"])
return contentKey, contentValue, nil
}
func clearNodeData(nodeDataDir string) {
err := os.Remove(fmt.Sprintf("%s%s", nodeDataDir, dbName))
if err != nil {
fmt.Println(err)
}
}

2
portalnetwork/beacon/testdata/light_client_bootstrap.json vendored
View file

File diff suppressed because one or more lines are too long

2
portalnetwork/beacon/testdata/light_client_finality_update.json vendored
View file

@ -1,6 +1,6 @@
{
"6718463": {
"content_key": "0x020000000000000000",
"content_key": "0x12a083660000000000",
"content_value": "0xbba4da9670010000b30400001d34030000000000000000000000000000000000000000000000000000000000a4f8b2415bbca66d73597343afead504bd8282523df27153543b2de8973b7474ace652bb73991c3b63f7185a17333c8aea619a69b69863a5d79810dad6c363d3df460765de20d3c8f56bcd9490f15e2d9e122b3f6173cea76f6d0d16074ce0f0ecde5df1b81f36c08e91ce803c25add312bfd8cb759a81b7cfd158d724959cf22e83908e043f77bdc5992806617ef1fa0a4f1985d9aa1b67371d3580be8c2c64ffffffffffffffffffffffffffffbfffff7fffffffffffffffffffffffffffffffffffffffffffffffffffdefffffffffffffffffffeffffffffffffffffefffa480f81d481c5aa8439f8c65f86c69d3b1570cfd95b61300f7e29c5b0954c0c0fb080bba0e8ba60dead40c45355332ed06982fcaf53ad1ad4a7e6e39091ee7e5602d4a30b6d46e12047c31451f246bf43c4a3fbdb1d1de2f2180a776ef71fc3d0084660000000000ff83660000000000e5f4020000000000007b8211614b9c5331a1de2691ad7cdde1ca113b4e51b9f2757a4b502833f9234a06eb807d82b9175dd085748ade76aaa86fb4eca48bc8deb01b9c56a267a896d98469820ef7d34f8610e134c6eaf8d379e83cf6c3ab0663fba828edab344dc5f40000008e2a532cfdfdce86b27b26f611f1df0e2b00a54d18c6980e69ad28a4b70bf480336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71b4c4e01065cd22a0d2f27a544ff8315b9e28f6270a63f543831778f32648ac3a15ecbea44f91b9b4a28ec8308a30a4b6252f554a2088bb1c73ea01013ba4963dfeebabe6b0418ec13b30aadf129f5dcdd4f70ceaa56499df0b3ba9d8fe6ca410132a29cf387371294137003adb139ce9f06b6f7dbf32ee01da91016a485a8aab5669168efc3d3a200b2ee0f25d937a413dfc38e44e211f02432a01a83088e700808c302002139aad8496010300cd220b14288429d8a6049945000a3b60f00f00109019341a0129a08906314848650817002cb98894a328084521a87e6d0a64e8080004ac0805000800e29808294016508a2004001304002a02320c4705501ac008a62848701040012a08e5c8b69010f1009f80408894857053e1b940e2c9004912860e26112080610d002098242000ca001420209a42c141d0c86087509344ced8a32e0204410a0196025a12106104e20008ad4449d5043208090008810201820209108607a0fa1004aa323410050222867024e0d172a82821b02c00020513152006484c800b10f0420802c0244d0850288c6541dc34cabc32f5d355aef4017425cab95c1de0474d483623e23c9bf13d9a18f1597e930b010000000080c3c901000000000ca88800000000004b6094640000000038020000ab9bf66d04000000000000000000000000000000000000000000000000000000797be971185ac699fa7288c87457c3611b443fb98483fddf282dfa7c5b4b27058eddbcd8b5de9b747c57c827ceba820d19e987ff6740eed38ccbc947696d71e3e2c3b96c9200d7d5c66fb4e8458067d6420e64647682ca2f0798ae817d2bc19468747470733a2f2f6574682d6275696c6465722e636f6da08366000000000037b10700000000002a7315c8ddfc25dc2266a6b221cb8f9fdf641970ab1f65a2754df4e14c432b9c446c604131913bed45976c4a8ea27df843a72d622f80ef15da622f0d56ec1ffe3021190177b0405c4fa1a0311a493c4dd095f24d386be6a03f5838a6b1008665f4000000f5c98fc152bf40e1ce216c8839b8ddd42ea5b355f0b5c700fc9b2cb7c802e1c8336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71a533e05d648cb9635382f38b778dc3c76b18e4eab2d017fdb24c1c0e32d2991707a45ce3f26e443dafa697a76bf24122c9b19b57eebe798668dbd1a0da7c900795222290dd7278aa3ddd389cc1e1d165cc4bafe5c658153cdef5850f97c4d4fb6706b82310414a00fe7dc34b3043e0c7a8a24e810e3cec9de3e1dd7ce06bc191efd2273a357da4436fc4bf82069f6cb87d3d605aeda1336ac513f358b4296b8da51232896ad271063d93404eaa6b80960fa2d4b41d74a544eef94731f0b83ba2701e33b49e3191098c3369bc4658d82f536cb9d38285c51c7844292ced06dd2e8148f0bcd585dc1104e64aa930045e83d828ab436a0e587637e5ef83044d1c2918da284d7072847132885d443640979907485847368e32501345b0dc24c904c06732cc17d944b26181b0c05c6461bccbe4f7b2180a282b6871aa6472fa36cdc49ec50cf69d3eee4b26fe02bf48f39efb549d3963d19e4112c7250c580543516204ff1b0675680cfea02ac09f0b23d6e28c14b08421b3a10a9788442457d732b120c5c466c54ad938dedd3cd3701d9b32fa15dc8db3d6b1ea5d8495732477a72b34efcaf8c349aa40d8e1f21f226ca11182cff00921930b010000000080c3c901000000009104e90000000000d75b9464000000003802000076ab1c8604000000000000000000000000000000000000000000000000000000844f7a7e7585362114fc88a70654146215ab4eaad65ad54b64975840b78d365e8b75e78616a92294e747589c3ee4a845c2962e1cf6cd7fe8bfbd09439e5e6d5e4d4efdf181473803e734d97891388e1b50628658ac818599cf2cb069c8e6b38b6265617665726275696c642e6f7267"
}
}

2
portalnetwork/beacon/testdata/light_client_optimistic_update.json vendored
View file

@ -1,6 +1,6 @@
{
"6718463": {
"content_key": "0x030000000000000000",
"content_key": "0x130084660000000000",
"content_value": "0xbba4da96ac000000ffffffffffffffffffffffffffffbfffff7fffffffffffffffffffffffffffffffffffffffffffffffffffdefffffffffffffffffffeffffffffffffffffefffa480f81d481c5aa8439f8c65f86c69d3b1570cfd95b61300f7e29c5b0954c0c0fb080bba0e8ba60dead40c45355332ed06982fcaf53ad1ad4a7e6e39091ee7e5602d4a30b6d46e12047c31451f246bf43c4a3fbdb1d1de2f2180a776ef71fc3d0084660000000000ff83660000000000e5f4020000000000007b8211614b9c5331a1de2691ad7cdde1ca113b4e51b9f2757a4b502833f9234a06eb807d82b9175dd085748ade76aaa86fb4eca48bc8deb01b9c56a267a896d98469820ef7d34f8610e134c6eaf8d379e83cf6c3ab0663fba828edab344dc5f40000008e2a532cfdfdce86b27b26f611f1df0e2b00a54d18c6980e69ad28a4b70bf480336488033fe5f3ef4ccc12af07b9370b92e553e35ecb4a337a1b1c0e4afe1e0edb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71b4c4e01065cd22a0d2f27a544ff8315b9e28f6270a63f543831778f32648ac3a15ecbea44f91b9b4a28ec8308a30a4b6252f554a2088bb1c73ea01013ba4963dfeebabe6b0418ec13b30aadf129f5dcdd4f70ceaa56499df0b3ba9d8fe6ca410132a29cf387371294137003adb139ce9f06b6f7dbf32ee01da91016a485a8aab5669168efc3d3a200b2ee0f25d937a413dfc38e44e211f02432a01a83088e700808c302002139aad8496010300cd220b14288429d8a6049945000a3b60f00f00109019341a0129a08906314848650817002cb98894a328084521a87e6d0a64e8080004ac0805000800e29808294016508a2004001304002a02320c4705501ac008a62848701040012a08e5c8b69010f1009f80408894857053e1b940e2c9004912860e26112080610d002098242000ca001420209a42c141d0c86087509344ced8a32e0204410a0196025a12106104e20008ad4449d5043208090008810201820209108607a0fa1004aa323410050222867024e0d172a82821b02c00020513152006484c800b10f0420802c0244d0850288c6541dc34cabc32f5d355aef4017425cab95c1de0474d483623e23c9bf13d9a18f1597e930b010000000080c3c901000000000ca88800000000004b6094640000000038020000ab9bf66d04000000000000000000000000000000000000000000000000000000797be971185ac699fa7288c87457c3611b443fb98483fddf282dfa7c5b4b27058eddbcd8b5de9b747c57c827ceba820d19e987ff6740eed38ccbc947696d71e3e2c3b96c9200d7d5c66fb4e8458067d6420e64647682ca2f0798ae817d2bc19468747470733a2f2f6574682d6275696c6465722e636f6d"
}
}

2
portalnetwork/beacon/testdata/light_client_updates_by_range.json vendored
View file

File diff suppressed because one or more lines are too long

15
portalnetwork/storage/config.go Normal file
View file

@ -0,0 +1,15 @@
package storage
import (
"database/sql"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/protolambda/zrnt/eth2/beacon/common"
)
type PortalStorageConfig struct {
StorageCapacityMB uint64
DB *sql.DB
NodeId enode.ID
Spec *common.Spec
}

24
portalnetwork/utils/util.go Normal file
View file

@ -0,0 +1,24 @@
package utils
import (
"errors"
"os"
)
func EnsureDir(dir string) error {
stat, err := os.Stat(dir)
if err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0755)
if err != nil {
return err
}
}
return err
}
if !stat.IsDir() {
return errors.New("node dir should be a dir")
}
return nil
}