From e622a2038e1dda72c39b7ae1c02af7a70a2881a3 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Thu, 20 Apr 2023 00:43:09 +0200 Subject: [PATCH] beacon/merkle: full implementation of compact proof format, simplified state proof db storage --- beacon/light/api/light_api.go | 44 +---- beacon/light/api/sync_server.go | 2 +- beacon/light/light_chain.go | 69 +++----- beacon/light/light_chain_test.go | 10 +- beacon/light/sync/state_sync.go | 14 +- beacon/light/types/header.go | 6 +- beacon/merkle/binary_merkle.go | 132 --------------- beacon/merkle/compact_proof.go | 271 +++++++++++++++++++++++++++++++ cmd/blsync/block_sync.go | 7 - cmd/blsync/main.go | 12 +- 10 files changed, 326 insertions(+), 241 deletions(-) create mode 100644 beacon/merkle/compact_proof.go diff --git a/beacon/light/api/light_api.go b/beacon/light/api/light_api.go index 20873d5225..1f2acf8454 100755 --- a/beacon/light/api/light_api.go +++ b/beacon/light/api/light_api.go @@ -207,16 +207,8 @@ func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error return header, nil } -// does not verify state root -//TODO ... -/*func (api *BeaconLightApi) GetHeadStateProof(format merkle.ProofFormat) (merkle.MultiProof, error) { - encFormat, bitLength := EncodeCompactProofFormat(format) //TODO cache encoding? - return api.getStateProof("head", format, encFormat, bitLength) -}*/ - -func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.ProofFormat) (merkle.MultiProof, error) { - encFormat, bitLength := EncodeCompactProofFormat(format) //TODO cache encoding? - proof, err := api.getStateProof(stateRoot.Hex(), format, encFormat, bitLength) +func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.CompactProofFormat) (merkle.MultiProof, error) { + proof, err := api.getStateProof(stateRoot.Hex(), format) if err != nil { return merkle.MultiProof{}, err } @@ -226,12 +218,12 @@ func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.Pr return proof, nil } -func (api *BeaconLightApi) getStateProof(stateId string, format merkle.ProofFormat, encFormat []byte, bitLength int) (merkle.MultiProof, error) { - resp, err := api.httpGetf("/eth/v0/beacon/proof/state/%s?format=0x%x", stateId, encFormat) +func (api *BeaconLightApi) getStateProof(stateId string, format merkle.CompactProofFormat) (merkle.MultiProof, error) { + resp, err := api.httpGetf("/eth/v0/beacon/proof/state/%s?format=0x%x", stateId, format.Format) if err != nil { return merkle.MultiProof{}, err } - valueCount := (bitLength + 1) / 2 + valueCount := format.ValueCount() if len(resp) != valueCount*32 { return merkle.MultiProof{}, errors.New("Invalid state proof length") } @@ -242,32 +234,6 @@ func (api *BeaconLightApi) getStateProof(stateId string, format merkle.ProofForm return merkle.MultiProof{Format: format, Values: values}, nil } -// EncodeCompactProofFormat encodes a merkle.ProofFormat into a binary compact -// proof format. See description here: -// https://github.com/ChainSafe/consensus-specs/blob/feat/multiproof/ssz/merkle-proofs.md#compact-multiproofs -func EncodeCompactProofFormat(format merkle.ProofFormat) ([]byte, int) { - target := make([]byte, 0, 64) - var bitLength int - encodeProofFormatSubtree(format, &target, &bitLength) - return target, bitLength -} - -// encodeProofFormatSubtree recursively encodes a subtree of a proof format into -// binary compact format. -func encodeProofFormatSubtree(format merkle.ProofFormat, target *[]byte, bitLength *int) { - bytePtr, bitMask := *bitLength>>3, byte(128)>>(*bitLength&7) - *bitLength++ - if bytePtr == len(*target) { - *target = append(*target, byte(0)) - } - if left, right := format.Children(); left == nil { - (*target)[bytePtr] += bitMask - } else { - encodeProofFormatSubtree(left, target, bitLength) - encodeProofFormatSubtree(right, target, bitLength) - } -} - // GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint. func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*light.CheckpointData, error) { resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:]) diff --git a/beacon/light/api/sync_server.go b/beacon/light/api/sync_server.go index e156b4b6b6..47eb689e7f 100755 --- a/beacon/light/api/sync_server.go +++ b/beacon/light/api/sync_server.go @@ -145,7 +145,7 @@ func (s *SyncServer) BeaconStateTail() uint64 { return s.firstState } -func (s *SyncServer) RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(*merkle.MultiProof)) { +func (s *SyncServer) RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.CompactProofFormat, response func(*merkle.MultiProof)) { go func() { if proof, err := s.api.GetStateProof(stateRoot, format); err == nil { response(&proof) diff --git a/beacon/light/light_chain.go b/beacon/light/light_chain.go index cb9eb156fc..3910dd26a8 100644 --- a/beacon/light/light_chain.go +++ b/beacon/light/light_chain.go @@ -40,7 +40,7 @@ var ( var ( chainRangeKey = []byte("range-") // RLP(chainRangeData) headerKey = []byte("header-") // bigEndian64(slot) + blockRoot -> RLP(types.Header) - stateKey = []byte("state-") // bigEndian64(slot) + stateRoot -> RLP(stateProofData) + stateKey = []byte("state-") // bigEndian64(slot) + stateRoot -> RLP(merkle.MultiProof) canonicalKey = []byte("canonical-") // bigEndian64(slot) -> canonical root hashToSlotKey = []byte("hash2slot-") // blockRoot -> RLP(slot) ) @@ -61,11 +61,10 @@ type LightChain struct { stateHead, stateTail types.Header // state proofs of canonical headers are available in this section lastStoredRange chainRangeData - headerCache *lru.Cache[slotAndHash, types.Header] - canonicalCache *lru.Cache[uint64, common.Hash] - hashToSlotCache *lru.Cache[common.Hash, uint64] - stateCache *lru.Cache[slotAndHash, merkle.Values] - stateProofFormat merkle.ProofFormat //TODO slot/parentSlot dependent format + headerCache *lru.Cache[slotAndHash, types.Header] + canonicalCache *lru.Cache[uint64, common.Hash] + hashToSlotCache *lru.Cache[common.Hash, uint64] + stateProofCache *lru.Cache[slotAndHash, merkle.MultiProof] } type slotAndHash struct { @@ -80,20 +79,14 @@ type chainRangeData struct { StateHead, StateTail uint64 } -type stateProofData struct { - FormatId uint //TODO compact binary format? - Values merkle.Values -} - // NewLightChain creates a new LightChain and loads canonical chain info from the database. -func NewLightChain(db ethdb.KeyValueStore, stateProofFormat merkle.ProofFormat) *LightChain { +func NewLightChain(db ethdb.KeyValueStore) *LightChain { lc := &LightChain{ - db: db, - stateProofFormat: stateProofFormat, - headerCache: lru.NewCache[slotAndHash, types.Header](500), - canonicalCache: lru.NewCache[uint64, common.Hash](2000), - hashToSlotCache: lru.NewCache[common.Hash, uint64](2000), - stateCache: lru.NewCache[slotAndHash, merkle.Values](100), + db: db, + headerCache: lru.NewCache[slotAndHash, types.Header](500), + canonicalCache: lru.NewCache[uint64, common.Hash](2000), + hashToSlotCache: lru.NewCache[common.Hash, uint64](2000), + stateProofCache: lru.NewCache[slotAndHash, merkle.MultiProof](100), } lc.loadChainRange() return lc @@ -340,7 +333,7 @@ func (lc *LightChain) Prune(beforeSlot uint64, removeCanonical bool) { } } batch.Delete(key) - lc.stateCache.Remove(slotAndHash{slot: slot, hash: stateRoot}) + lc.stateProofCache.Remove(slotAndHash{slot: slot, hash: stateRoot}) } } @@ -355,7 +348,7 @@ func getHeaderKey(slot uint64, blockRoot common.Hash) []byte { return key } -func getStateKey(slot uint64, stateRoot common.Hash) []byte { +func getStateProofKey(slot uint64, stateRoot common.Hash) []byte { var ( kl = len(stateKey) key = make([]byte, kl+8+32) @@ -554,44 +547,39 @@ func (lc *LightChain) getSlotByHash(blockRoot common.Hash) (uint64, bool) { // HasStateProof returns true if a state proof belonging to the given header exists. func (lc *LightChain) HasStateProof(header types.Header) bool { - if _, ok := lc.stateCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok { + if _, ok := lc.stateProofCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok { return true } - ok, err := lc.db.Has(getStateKey(header.Slot, header.StateRoot)) + ok, err := lc.db.Has(getStateProofKey(header.Slot, header.StateRoot)) return ok && err == nil } // GetStateProof returns the state proof belonging to the given header. func (lc *LightChain) GetStateProof(header types.Header) (merkle.MultiProof, error) { - if values, ok := lc.stateCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok { - return merkle.MultiProof{Format: lc.stateProofFormat, Values: values}, nil + if proof, ok := lc.stateProofCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok { + return proof, nil } - stateEnc, err := lc.db.Get(getStateKey(header.Slot, header.StateRoot)) + proofEnc, err := lc.db.Get(getStateProofKey(header.Slot, header.StateRoot)) if err != nil { return merkle.MultiProof{}, ErrNotFound } - var state stateProofData - if err := rlp.DecodeBytes(stateEnc, &state); err != nil { + var proof merkle.MultiProof + if err := proof.Decode(proofEnc); err != nil { log.Error("Failed to decode state proof data", "error", err) return merkle.MultiProof{}, ErrNotFound } - return merkle.MultiProof{Format: lc.stateProofFormat, Values: state.Values}, nil -} - -// StateProofFormat returns the expected state proof format for the given header. -func (lc *LightChain) StateProofFormat(header types.Header) merkle.ProofFormat { - return lc.stateProofFormat + return proof, nil } // AddStateProof adds a state proof. If it belongs to a canonical header then // the state range is also updated. +// Note: it is the caller's responsibility to make sure that the proof has the +// right format for the given application; this function only verifies the state +// root against the corresponding header. func (lc *LightChain) AddStateProof(header types.Header, proof merkle.MultiProof) (err error) { lc.lock.Lock() defer lc.lock.Unlock() - if !merkle.IsEqual(proof.Format, lc.StateProofFormat(header)) { - return ErrInvalidProofFormat - } if proof.RootHash() != header.StateRoot { return ErrInvalidStateRoot } @@ -602,13 +590,8 @@ func (lc *LightChain) AddStateProof(header types.Header, proof merkle.MultiProof } }() - stateEnc, err := rlp.EncodeToBytes(&stateProofData{Values: proof.Values}) - if err != nil { - log.Error("Failed to encode state proof data", "error", err) - return err - } - batch.Put(getStateKey(header.Slot, header.StateRoot), stateEnc) - lc.stateCache.Add(slotAndHash{header.Slot, header.StateRoot}, proof.Values) + batch.Put(getStateProofKey(header.Slot, header.StateRoot), proof.Encode()) + lc.stateProofCache.Add(slotAndHash{header.Slot, header.StateRoot}, proof) if !lc.IsCanonical(header) { return nil } diff --git a/beacon/light/light_chain_test.go b/beacon/light/light_chain_test.go index 2ccd53934b..6c3c182ba9 100644 --- a/beacon/light/light_chain_test.go +++ b/beacon/light/light_chain_test.go @@ -230,7 +230,7 @@ func TestLightChainPrune(t *testing.T) { type lightChainTest struct { t *testing.T db *memorydb.Database - proofFormat merkle.ProofFormat + proofFormat merkle.CompactProofFormat chain *LightChain headers []types.Header // not added to the chain yet stateProofs []testProof // not added to the chain yet @@ -246,10 +246,10 @@ func newLightChainTest(t *testing.T) *lightChainTest { c := &lightChainTest{ t: t, db: memorydb.New(), - proofFormat: merkle.NewIndexMapFormat().AddLeaf(42, nil).AddLeaf(67, nil), + proofFormat: merkle.EncodeCompactProofFormat(merkle.NewIndexMapFormat().AddLeaf(42, nil).AddLeaf(67, nil)), emptyRatio: 20, } - c.chain = NewLightChain(c.db, c.proofFormat) + c.chain = NewLightChain(c.db) return c } @@ -292,12 +292,12 @@ func (c *lightChainTest) checkTail(header, expTail types.Header) { } func (c *lightChainTest) reloadChain() { - c.chain = NewLightChain(c.db, c.proofFormat) + c.chain = NewLightChain(c.db) } func (c *lightChainTest) makeChain(from types.Header, targetHeadSlot uint64, addHeaders, addStateProofs bool) (tail, head types.Header) { head = from - valueCount := merkle.ValueCount(c.proofFormat) + valueCount := c.proofFormat.ValueCount() for head.Slot < targetHeadSlot { var ( slot uint64 diff --git a/beacon/light/sync/state_sync.go b/beacon/light/sync/state_sync.go index d07f2cbbf7..92653b5b51 100644 --- a/beacon/light/sync/state_sync.go +++ b/beacon/light/sync/state_sync.go @@ -33,7 +33,7 @@ import ( type beaconStateServer interface { request.RequestServer BeaconStateTail() uint64 - RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(*merkle.MultiProof)) + RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.CompactProofFormat, response func(*merkle.MultiProof)) } type StateSync struct { @@ -41,16 +41,18 @@ type StateSync struct { reqLock request.MultiLock chain *light.LightChain prefetch bool + syncProofFormat merkle.CompactProofFormat targetTailSlot uint64 headSyncPossible uint32 selfTrigger, headStateTrigger *request.ModuleTrigger } -func NewStateSync(chain *light.LightChain, prefetch bool) *StateSync { +func NewStateSync(chain *light.LightChain, syncProofFormat merkle.CompactProofFormat, prefetch bool) *StateSync { return &StateSync{ - chain: chain, - prefetch: prefetch, - targetTailSlot: math.MaxUint64, + chain: chain, + syncProofFormat: syncProofFormat, + prefetch: prefetch, + targetTailSlot: math.MaxUint64, } } @@ -203,7 +205,7 @@ func (r stateRequest) CanSendTo(server *request.Server) (canSend bool, priority func (r stateRequest) SendTo(server *request.Server) { reqId := r.reqLock.Send(server, r.header.StateRoot) - server.RequestServer.(beaconStateServer).RequestBeaconState(r.header.Slot, r.header.StateRoot, r.chain.StateProofFormat(r.header), func(proof *merkle.MultiProof) { + server.RequestServer.(beaconStateServer).RequestBeaconState(r.header.Slot, r.header.StateRoot, r.syncProofFormat, func(proof *merkle.MultiProof) { r.lock.Lock() defer r.lock.Unlock() diff --git a/beacon/light/types/header.go b/beacon/light/types/header.go index cd4731ffb9..e9526b50e3 100644 --- a/beacon/light/types/header.go +++ b/beacon/light/types/header.go @@ -25,6 +25,8 @@ import ( "github.com/ethereum/go-ethereum/common" ) +var headerFormat = merkle.EncodeCompactProofFormat(merkle.NewRangeFormat(8, 15, nil)) + // Header defines a beacon header // // See data structure definition here: @@ -83,7 +85,7 @@ func (bh *Header) Hash() common.Hash { values[params.BhiParentRoot-8] = merkle.Value(bh.ParentRoot) values[params.BhiStateRoot-8] = merkle.Value(bh.StateRoot) values[params.BhiBodyRoot-8] = merkle.Value(bh.BodyRoot) - return merkle.MultiProof{Format: merkle.NewRangeFormat(8, 15, nil), Values: values[:]}.RootHash() + return merkle.MultiProof{Format: headerFormat, Values: values[:]}.RootHash() } // Epoch returns the epoch the header belongs to @@ -127,7 +129,7 @@ func (bh *HeaderWithoutState) Proof(stateRoot common.Hash) merkle.MultiProof { values[params.BhiParentRoot-8] = merkle.Value(bh.ParentRoot) values[params.BhiStateRoot-8] = merkle.Value(stateRoot) values[params.BhiBodyRoot-8] = merkle.Value(bh.BodyRoot) - return merkle.MultiProof{Format: merkle.NewRangeFormat(8, 15, nil), Values: values[:]} + return merkle.MultiProof{Format: headerFormat, Values: values[:]} } // FullHeader reconstructs a full Header from a HeaderWithoutState and a state root diff --git a/beacon/merkle/binary_merkle.go b/beacon/merkle/binary_merkle.go index b3123b0b48..9a9ecb6be0 100644 --- a/beacon/merkle/binary_merkle.go +++ b/beacon/merkle/binary_merkle.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/log" "github.com/minio/sha256-simd" ) @@ -69,16 +68,6 @@ type ProofFormat interface { Children() (left, right ProofFormat) // either both or neither should be nil } -// IsEqual returns true if the two formats are the same -func IsEqual(a, b ProofFormat) bool { - al, ar := a.Children() - bl, br := b.Children() - if al == nil || bl == nil { - return al == nil && bl == nil - } - return IsEqual(al, bl) && IsEqual(ar, br) -} - // ValueCount returns the number of merkle values required for this proof format func ValueCount(f ProofFormat) int { if f == nil { @@ -147,127 +136,6 @@ func TraverseProof(reader ProofReader, writer ProofWriter) (common.Hash, bool) { return common.Hash(node), true } -// MultiProof stores a partial Merkle tree proof -type MultiProof struct { - Format ProofFormat - Values Values -} - -// multiProofReader implements ProofReader based on a MultiProof and also allows -// attaching further subtree readers at certain indices -// Note: valuePtr is stored and copied as a reference because child readers read -// from the same value list as the tree is traversed -type multiProofReader struct { - format ProofFormat // corresponding proof format - values Values // proof values - valuePtr *int // next index to be read from values - index uint64 // generalized tree index - subtrees func(uint64) ProofReader // attached subtrees -} - -// children implements ProofReader -func (mpr multiProofReader) Children() (left, right ProofReader) { - lf, rf := mpr.format.Children() - if lf == nil { - if mpr.subtrees != nil { - if subtree := mpr.subtrees(mpr.index); subtree != nil { - return subtree.Children() - } - } - return nil, nil - } - return multiProofReader{format: lf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index * 2, subtrees: mpr.subtrees}, - multiProofReader{format: rf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index*2 + 1, subtrees: mpr.subtrees} -} - -// readNode implements ProofReader -func (mpr multiProofReader) ReadNode() (Value, bool) { - if l, _ := mpr.format.Children(); l == nil && len(mpr.values) > *mpr.valuePtr { - hash := mpr.values[*mpr.valuePtr] - (*mpr.valuePtr)++ - return hash, true - } - return Value{}, false -} - -// Reader creates a multiProofReader for the given proof; if subtrees != nil -// then also attaches subtree readers at indices where the function returns a -// non-nil reader. -// Note that the reader can only be traversed once as the values slice is -// sequentially consumed. -func (mp MultiProof) Reader(subtrees func(uint64) ProofReader) multiProofReader { - return multiProofReader{format: mp.Format, values: mp.Values, valuePtr: new(int), index: 1, subtrees: subtrees} -} - -// Finished returns true if all values have been consumed by the traversal. -// Should be checked after TraverseProof if received from an untrusted source in -// order to prevent DoS attacks by excess proof values. -func (mpr multiProofReader) Finished() bool { - return len(mpr.values) == *mpr.valuePtr -} - -// rootHash returns the root hash of the proven structure. -func (mp MultiProof) RootHash() common.Hash { - reader := mp.Reader(nil) - hash, ok := TraverseProof(reader, nil) - if !ok || !reader.Finished() { - log.Error("MultiProof.rootHash: invalid proof format") - } - return hash -} - -// multiProofWriter implements ProofWriter and creates a MultiProof with the -// previously specified format. Also allows attaching further subtree writers at -// certain indices. -// Note: values is stored and copied as a reference because child writers append -// to the same value list as the tree is traversed -type multiProofWriter struct { - format ProofFormat // target proof format - values *Values // target proof value list - index uint64 // generalized tree index - subtrees func(uint64) ProofWriter // attached subtrees -} - -// NewMultiProofWriter creates a new multiproof writer with the specified format. -// If subtrees != nil then further subtree writers are attached at indices where -// the function returns a non-nil writer. -// Note that the specified format should not include these attached subtrees; -// they should be attached at leaf indices of the given format. -// Also note that target can be nil in which case the nodes specified by the format -// are traversed but not stored; subtree writers might still store tree data. -func NewMultiProofWriter(format ProofFormat, target *Values, subtrees func(uint64) ProofWriter) multiProofWriter { - return multiProofWriter{format: format, values: target, index: 1, subtrees: subtrees} -} - -// children implements ProofWriter -func (mpw multiProofWriter) Children() (left, right ProofWriter) { - if mpw.subtrees != nil { - if subtree := mpw.subtrees(mpw.index); subtree != nil { - return subtree.Children() - } - } - lf, rf := mpw.format.Children() - if lf == nil { - return nil, nil - } - return multiProofWriter{format: lf, values: mpw.values, index: mpw.index * 2, subtrees: mpw.subtrees}, - multiProofWriter{format: rf, values: mpw.values, index: mpw.index*2 + 1, subtrees: mpw.subtrees} -} - -// writeNode implements ProofWriter -func (mpw multiProofWriter) WriteNode(node Value) { - if mpw.values != nil { - if lf, _ := mpw.format.Children(); lf == nil { - *mpw.values = append(*mpw.values, node) - } - } - if mpw.subtrees != nil { - if subtree := mpw.subtrees(mpw.index); subtree != nil { - subtree.WriteNode(node) - } - } -} - // ProofFormatIndexMap creates a generalized tree index -> MultiProof value // slice index association map based on the given proof format. func ProofFormatIndexMap(f ProofFormat) map[uint64]int { diff --git a/beacon/merkle/compact_proof.go b/beacon/merkle/compact_proof.go new file mode 100644 index 0000000000..3087d34270 --- /dev/null +++ b/beacon/merkle/compact_proof.go @@ -0,0 +1,271 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package merkle + +import ( + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +// CompactProofFormat is a binary compact proof format, see description here: +// https://github.com/ChainSafe/consensus-specs/blob/feat/multiproof/ssz/merkle-proofs.md#compact-multiproofs +type CompactProofFormat struct { + Format []byte + firstBit, afterLastBit int +} + +// Children implements ProofFormat +func (c CompactProofFormat) Children() (left, right ProofFormat) { + if bit, ok := c.readFirstBit(); !ok { + log.Error("Invalid compact proof format") + } else if bit { + return + } + l, r := c, c + if !r.skipSubtree() { + log.Error("Invalid compact proof format") + } + return l, r +} + +// ValueCount returns the number of merkle values required for this proof +func (c CompactProofFormat) ValueCount() int { + return (c.afterLastBit + 1 - c.firstBit) / 2 +} + +// EncodeCompactProofFormat encodes a ProofFormat into a binary compact proof format. +func EncodeCompactProofFormat(format ProofFormat) (c CompactProofFormat) { + c.encodeFormatSubtree(format) + return +} + +// readFirstBit reads the first bit of the bit vector and moves the first bit +// pointer one bit ahead. +func (c *CompactProofFormat) readFirstBit() (bit, ok bool) { + if c.firstBit >= c.afterLastBit { + return false, false + } + bit = c.Format[c.firstBit>>3]&(byte(128)>>(c.firstBit&7)) != 0 + c.firstBit++ + return bit, true +} + +// skipSubtree moves the first bit pointer beyond the subtree it was pointing at +// before and returns true if successful. +func (c *CompactProofFormat) skipSubtree() bool { + if bit, ok := c.readFirstBit(); !ok { + return false + } else if bit { + return true + } + return c.skipSubtree() && c.skipSubtree() +} + +// appendBit adds a bit at the end of the bit vector. +func (c *CompactProofFormat) appendBit(bit bool) { + bytePtr := c.afterLastBit >> 3 + if bytePtr == len(c.Format) { + c.Format = append(c.Format, byte(0)) + } + if bit { + c.Format[bytePtr] += byte(128) >> (c.afterLastBit & 7) + } + c.afterLastBit++ +} + +// encodeFormatSubtree encodes a ProofFormat subtree at the end of the bit vector. +func (c *CompactProofFormat) encodeFormatSubtree(format ProofFormat) { + if left, right := format.Children(); left == nil { + c.appendBit(true) + } else { + c.appendBit(false) + c.encodeFormatSubtree(left) + c.encodeFormatSubtree(right) + } +} + +// encodeProofFormatSubtree recursively encodes a subtree of a proof format into +// binary compact format. +func encodeProofFormatSubtree(format ProofFormat, target *[]byte, bitLength *int) { + bytePtr, bitMask := *bitLength>>3, byte(128)>>(*bitLength&7) + *bitLength++ + if bytePtr == len(*target) { + *target = append(*target, byte(0)) + } + if left, right := format.Children(); left == nil { + (*target)[bytePtr] += bitMask + } else { + encodeProofFormatSubtree(left, target, bitLength) + encodeProofFormatSubtree(right, target, bitLength) + } +} + +// MultiProof stores a partial Merkle tree proof +type MultiProof struct { + Format CompactProofFormat + Values Values +} + +// Encode encodes a MultiProof into a byte slice +func (m *MultiProof) Encode() []byte { + lf := len(m.Format.Format) + enc := make([]byte, lf+32*len(m.Values)) + copy(enc[:lf], m.Format.Format) + for i, value := range m.Values { + copy(enc[lf+i*32:lf+(i+1)*32], value[:]) + } + return enc +} + +// Decode decodes a MultiProof from a byte slice +func (m *MultiProof) Decode(enc []byte) error { + valueCount := len(enc) * 4 / 129 + lf := (valueCount + 3) / 4 + if len(enc) != lf+32*valueCount { + return errors.New("Invalid length for encoded MultiProof") + } + format := CompactProofFormat{ + Format: make([]byte, lf), + afterLastBit: valueCount*2 - 1, + } + copy(format.Format, enc[:lf]) + if f := format; !f.skipSubtree() || f.firstBit != f.afterLastBit { + log.Error("Invalid compact proof format") + } + m.Format, m.Values = format, make(Values, valueCount) + for i := range m.Values { + copy(m.Values[i][:], enc[lf+i*32:lf+(i+1)*32]) + } + return nil +} + +// multiProofReader implements ProofReader based on a MultiProof and also allows +// attaching further subtree readers at certain indices +// Note: valuePtr is stored and copied as a reference because child readers read +// from the same value list as the tree is traversed +type multiProofReader struct { + format ProofFormat // corresponding proof format + values Values // proof values + valuePtr *int // next index to be read from values + index uint64 // generalized tree index + subtrees func(uint64) ProofReader // attached subtrees +} + +// children implements ProofReader +func (mpr multiProofReader) Children() (left, right ProofReader) { + lf, rf := mpr.format.Children() + if lf == nil { + if mpr.subtrees != nil { + if subtree := mpr.subtrees(mpr.index); subtree != nil { + return subtree.Children() + } + } + return nil, nil + } + return multiProofReader{format: lf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index * 2, subtrees: mpr.subtrees}, + multiProofReader{format: rf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index*2 + 1, subtrees: mpr.subtrees} +} + +// readNode implements ProofReader +func (mpr multiProofReader) ReadNode() (Value, bool) { + if l, _ := mpr.format.Children(); l == nil && len(mpr.values) > *mpr.valuePtr { + hash := mpr.values[*mpr.valuePtr] + (*mpr.valuePtr)++ + return hash, true + } + return Value{}, false +} + +// Reader creates a multiProofReader for the given proof; if subtrees != nil +// then also attaches subtree readers at indices where the function returns a +// non-nil reader. +// Note that the reader can only be traversed once as the values slice is +// sequentially consumed. +func (mp MultiProof) Reader(subtrees func(uint64) ProofReader) multiProofReader { + return multiProofReader{format: mp.Format, values: mp.Values, valuePtr: new(int), index: 1, subtrees: subtrees} +} + +// Finished returns true if all values have been consumed by the traversal. +// Should be checked after TraverseProof if received from an untrusted source in +// order to prevent DoS attacks by excess proof values. +func (mpr multiProofReader) Finished() bool { + return len(mpr.values) == *mpr.valuePtr +} + +// rootHash returns the root hash of the proven structure. +func (mp MultiProof) RootHash() common.Hash { + reader := mp.Reader(nil) + hash, ok := TraverseProof(reader, nil) + if !ok || !reader.Finished() { + log.Error("MultiProof.rootHash: invalid proof format") + } + return hash +} + +// multiProofWriter implements ProofWriter and creates a MultiProof with the +// previously specified format. Also allows attaching further subtree writers at +// certain indices. +// Note: values is stored and copied as a reference because child writers append +// to the same value list as the tree is traversed +type multiProofWriter struct { + format ProofFormat // target proof format + values *Values // target proof value list + index uint64 // generalized tree index + subtrees func(uint64) ProofWriter // attached subtrees +} + +// NewMultiProofWriter creates a new multiproof writer with the specified format. +// If subtrees != nil then further subtree writers are attached at indices where +// the function returns a non-nil writer. +// Note that the specified format should not include these attached subtrees; +// they should be attached at leaf indices of the given format. +// Also note that target can be nil in which case the nodes specified by the format +// are traversed but not stored; subtree writers might still store tree data. +func NewMultiProofWriter(format ProofFormat, target *Values, subtrees func(uint64) ProofWriter) multiProofWriter { + return multiProofWriter{format: format, values: target, index: 1, subtrees: subtrees} +} + +// children implements ProofWriter +func (mpw multiProofWriter) Children() (left, right ProofWriter) { + if mpw.subtrees != nil { + if subtree := mpw.subtrees(mpw.index); subtree != nil { + return subtree.Children() + } + } + lf, rf := mpw.format.Children() + if lf == nil { + return nil, nil + } + return multiProofWriter{format: lf, values: mpw.values, index: mpw.index * 2, subtrees: mpw.subtrees}, + multiProofWriter{format: rf, values: mpw.values, index: mpw.index*2 + 1, subtrees: mpw.subtrees} +} + +// writeNode implements ProofWriter +func (mpw multiProofWriter) WriteNode(node Value) { + if mpw.values != nil { + if lf, _ := mpw.format.Children(); lf == nil { + *mpw.values = append(*mpw.values, node) + } + } + if mpw.subtrees != nil { + if subtree := mpw.subtrees(mpw.index); subtree != nil { + subtree.WriteNode(node) + } + } +} diff --git a/cmd/blsync/block_sync.go b/cmd/blsync/block_sync.go index e3352b4041..6bc807fd29 100755 --- a/cmd/blsync/block_sync.go +++ b/cmd/blsync/block_sync.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/beacon/light/request" lsync "github.com/ethereum/go-ethereum/beacon/light/sync" "github.com/ethereum/go-ethereum/beacon/light/types" - "github.com/ethereum/go-ethereum/beacon/merkle" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" ctypes "github.com/ethereum/go-ethereum/core/types" @@ -46,12 +45,6 @@ type beaconBlockServer interface { RequestBeaconBlock(blockRoot common.Hash, response func(*capella.BeaconBlock)) } -type beaconStateServer interface { - request.RequestServer - BeaconStateTail() uint64 - RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(merkle.MultiProof)) -} - type beaconBlockSync struct { lock sync.Mutex reqLock request.MultiLock diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go index 8f73e2888a..492b4ba2f3 100644 --- a/cmd/blsync/main.go +++ b/cmd/blsync/main.go @@ -70,16 +70,16 @@ func main() { } var ( - stateProofFormat merkle.ProofFormat // requested multiproof format - execBlockIndex int // index of execution block root in proof.Values where proof.Format == stateProofFormat - finalizedBlockIndex int // index of finalized block root in proof.Values where proof.Format == stateProofFormat + stateProofFormat merkle.CompactProofFormat // requested multiproof format + execBlockIndex int // index of execution block root in proof.Values where proof.Format == stateProofFormat + finalizedBlockIndex int // index of finalized block root in proof.Values where proof.Format == stateProofFormat ) func blsync(ctx *cli.Context) error { if !ctx.IsSet(utils.BeaconApiFlag.Name) { utils.Fatalf("Beacon node light client API URL not specified") } - stateProofFormat = merkle.NewIndexMapFormat().AddLeaf(params.BsiExecHead, nil).AddLeaf(params.BsiFinalBlock, nil) + stateProofFormat = merkle.EncodeCompactProofFormat(merkle.NewIndexMapFormat().AddLeaf(params.BsiExecHead, nil).AddLeaf(params.BsiFinalBlock, nil)) var ( stateIndexMap = merkle.ProofFormatIndexMap(stateProofFormat) chainConfig = makeChainConfig(ctx) @@ -103,7 +103,7 @@ func blsync(ctx *cli.Context) error { committeeChain = light.NewCommitteeChain(db, chainConfig.Forks, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name), light.BLSVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() }) checkpointStore = light.NewCheckpointStore(db, committeeChain) headValidator = light.NewHeadValidator(committeeChain) - lightChain = light.NewLightChain(db, stateProofFormat) + lightChain = light.NewLightChain(db) ) committeeChain.SetGenesisData(chainConfig.GenesisData) headUpdater := sync.NewHeadUpdater(headValidator, committeeChain) @@ -116,7 +116,7 @@ func blsync(ctx *cli.Context) error { checkpointInit := sync.NewCheckpointInit(committeeChain, checkpointStore, chainConfig.Checkpoint) forwardSync := sync.NewForwardUpdateSync(committeeChain) headerSync := sync.NewHeaderSync(lightChain, false) - stateSync := sync.NewStateSync(lightChain, true) + stateSync := sync.NewStateSync(lightChain, stateProofFormat, true) beaconBlockSync := newBeaconBlockSyncer(lightChain) engineApiUpdater := &engineApiUpdater{ //TODO constructor client: makeRPCClient(ctx),