beacon/merkle: full implementation of compact proof format, simplified state proof db storage

beacon/light/api/light_api.go

@@ -207,16 +207,8 @@ func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error
 	return header, nil
 }
 
-// does not verify state root
+func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.CompactProofFormat) (merkle.MultiProof, error) {
-//TODO ...
+	proof, err := api.getStateProof(stateRoot.Hex(), format)
-/*func (api *BeaconLightApi) GetHeadStateProof(format merkle.ProofFormat) (merkle.MultiProof, error) {
-	encFormat, bitLength := EncodeCompactProofFormat(format) //TODO cache encoding?
-	return api.getStateProof("head", format, encFormat, bitLength)
-}*/
-
-func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.ProofFormat) (merkle.MultiProof, error) {
-	encFormat, bitLength := EncodeCompactProofFormat(format) //TODO cache encoding?
-	proof, err := api.getStateProof(stateRoot.Hex(), format, encFormat, bitLength)
 	if err != nil {
 		return merkle.MultiProof{}, err
 	}
@@ -226,12 +218,12 @@ func (api *BeaconLightApi) GetStateProof(stateRoot common.Hash, format merkle.Pr
 	return proof, nil
 }
 
-func (api *BeaconLightApi) getStateProof(stateId string, format merkle.ProofFormat, encFormat []byte, bitLength int) (merkle.MultiProof, error) {
+func (api *BeaconLightApi) getStateProof(stateId string, format merkle.CompactProofFormat) (merkle.MultiProof, error) {
-	resp, err := api.httpGetf("/eth/v0/beacon/proof/state/%s?format=0x%x", stateId, encFormat)
+	resp, err := api.httpGetf("/eth/v0/beacon/proof/state/%s?format=0x%x", stateId, format.Format)
 	if err != nil {
 		return merkle.MultiProof{}, err
 	}
-	valueCount := (bitLength + 1) / 2
+	valueCount := format.ValueCount()
 	if len(resp) != valueCount*32 {
 		return merkle.MultiProof{}, errors.New("Invalid state proof length")
 	}
@@ -242,32 +234,6 @@ func (api *BeaconLightApi) getStateProof(stateId string, format merkle.ProofForm
 	return merkle.MultiProof{Format: format, Values: values}, nil
 }
 
-// EncodeCompactProofFormat encodes a merkle.ProofFormat into a binary compact
-// proof format. See description here:
-// https://github.com/ChainSafe/consensus-specs/blob/feat/multiproof/ssz/merkle-proofs.md#compact-multiproofs
-func EncodeCompactProofFormat(format merkle.ProofFormat) ([]byte, int) {
-	target := make([]byte, 0, 64)
-	var bitLength int
-	encodeProofFormatSubtree(format, &target, &bitLength)
-	return target, bitLength
-}
-
-// encodeProofFormatSubtree recursively encodes a subtree of a proof format into
-// binary compact format.
-func encodeProofFormatSubtree(format merkle.ProofFormat, target *[]byte, bitLength *int) {
-	bytePtr, bitMask := *bitLength>>3, byte(128)>>(*bitLength&7)
-	*bitLength++
-	if bytePtr == len(*target) {
-		*target = append(*target, byte(0))
-	}
-	if left, right := format.Children(); left == nil {
-		(*target)[bytePtr] += bitMask
-	} else {
-		encodeProofFormatSubtree(left, target, bitLength)
-		encodeProofFormatSubtree(right, target, bitLength)
-	}
-}
-
 // GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint.
 func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*light.CheckpointData, error) {
 	resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:])

beacon/light/api/sync_server.go

@@ -145,7 +145,7 @@ func (s *SyncServer) BeaconStateTail() uint64 {
 	return s.firstState
 }
 
-func (s *SyncServer) RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(*merkle.MultiProof)) {
+func (s *SyncServer) RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.CompactProofFormat, response func(*merkle.MultiProof)) {
 	go func() {
 		if proof, err := s.api.GetStateProof(stateRoot, format); err == nil {
 			response(&proof)

beacon/light/light_chain.go

@@ -40,7 +40,7 @@ var (
 var (
 	chainRangeKey = []byte("range-")     // RLP(chainRangeData)
 	headerKey     = []byte("header-")    // bigEndian64(slot) + blockRoot -> RLP(types.Header)
-	stateKey      = []byte("state-")     // bigEndian64(slot) + stateRoot -> RLP(stateProofData)
+	stateKey      = []byte("state-")     // bigEndian64(slot) + stateRoot -> RLP(merkle.MultiProof)
 	canonicalKey  = []byte("canonical-") // bigEndian64(slot) -> canonical root
 	hashToSlotKey = []byte("hash2slot-") // blockRoot -> RLP(slot)
 )
@@ -61,11 +61,10 @@ type LightChain struct {
 	stateHead, stateTail types.Header // state proofs of canonical headers are available in this section
 	lastStoredRange      chainRangeData
 
-	headerCache      *lru.Cache[slotAndHash, types.Header]
+	headerCache     *lru.Cache[slotAndHash, types.Header]
-	canonicalCache   *lru.Cache[uint64, common.Hash]
+	canonicalCache  *lru.Cache[uint64, common.Hash]
-	hashToSlotCache  *lru.Cache[common.Hash, uint64]
+	hashToSlotCache *lru.Cache[common.Hash, uint64]
-	stateCache       *lru.Cache[slotAndHash, merkle.Values]
+	stateProofCache *lru.Cache[slotAndHash, merkle.MultiProof]
-	stateProofFormat merkle.ProofFormat //TODO slot/parentSlot dependent format
 }
 
 type slotAndHash struct {
@@ -80,20 +79,14 @@ type chainRangeData struct {
 	StateHead, StateTail uint64
 }
 
-type stateProofData struct {
-	FormatId uint //TODO compact binary format?
-	Values   merkle.Values
-}
-
 // NewLightChain creates a new LightChain and loads canonical chain info from the database.
-func NewLightChain(db ethdb.KeyValueStore, stateProofFormat merkle.ProofFormat) *LightChain {
+func NewLightChain(db ethdb.KeyValueStore) *LightChain {
 	lc := &LightChain{
-		db:               db,
+		db:              db,
-		stateProofFormat: stateProofFormat,
+		headerCache:     lru.NewCache[slotAndHash, types.Header](500),
-		headerCache:      lru.NewCache[slotAndHash, types.Header](500),
+		canonicalCache:  lru.NewCache[uint64, common.Hash](2000),
-		canonicalCache:   lru.NewCache[uint64, common.Hash](2000),
+		hashToSlotCache: lru.NewCache[common.Hash, uint64](2000),
-		hashToSlotCache:  lru.NewCache[common.Hash, uint64](2000),
+		stateProofCache: lru.NewCache[slotAndHash, merkle.MultiProof](100),
-		stateCache:       lru.NewCache[slotAndHash, merkle.Values](100),
 	}
 	lc.loadChainRange()
 	return lc
@@ -340,7 +333,7 @@ func (lc *LightChain) Prune(beforeSlot uint64, removeCanonical bool) {
 			}
 		}
 		batch.Delete(key)
-		lc.stateCache.Remove(slotAndHash{slot: slot, hash: stateRoot})
+		lc.stateProofCache.Remove(slotAndHash{slot: slot, hash: stateRoot})
 	}
 }
 
@@ -355,7 +348,7 @@ func getHeaderKey(slot uint64, blockRoot common.Hash) []byte {
 	return key
 }
 
-func getStateKey(slot uint64, stateRoot common.Hash) []byte {
+func getStateProofKey(slot uint64, stateRoot common.Hash) []byte {
 	var (
 		kl  = len(stateKey)
 		key = make([]byte, kl+8+32)
@@ -554,44 +547,39 @@ func (lc *LightChain) getSlotByHash(blockRoot common.Hash) (uint64, bool) {
 
 // HasStateProof returns true if a state proof belonging to the given header exists.
 func (lc *LightChain) HasStateProof(header types.Header) bool {
-	if _, ok := lc.stateCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok {
+	if _, ok := lc.stateProofCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok {
 		return true
 	}
-	ok, err := lc.db.Has(getStateKey(header.Slot, header.StateRoot))
+	ok, err := lc.db.Has(getStateProofKey(header.Slot, header.StateRoot))
 	return ok && err == nil
 }
 
 // GetStateProof returns the state proof belonging to the given header.
 func (lc *LightChain) GetStateProof(header types.Header) (merkle.MultiProof, error) {
-	if values, ok := lc.stateCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok {
+	if proof, ok := lc.stateProofCache.Get(slotAndHash{header.Slot, header.StateRoot}); ok {
-		return merkle.MultiProof{Format: lc.stateProofFormat, Values: values}, nil
+		return proof, nil
 	}
-	stateEnc, err := lc.db.Get(getStateKey(header.Slot, header.StateRoot))
+	proofEnc, err := lc.db.Get(getStateProofKey(header.Slot, header.StateRoot))
 	if err != nil {
 		return merkle.MultiProof{}, ErrNotFound
 	}
-	var state stateProofData
+	var proof merkle.MultiProof
-	if err := rlp.DecodeBytes(stateEnc, &state); err != nil {
+	if err := proof.Decode(proofEnc); err != nil {
 		log.Error("Failed to decode state proof data", "error", err)
 		return merkle.MultiProof{}, ErrNotFound
 	}
-	return merkle.MultiProof{Format: lc.stateProofFormat, Values: state.Values}, nil
+	return proof, nil
-}
-
-// StateProofFormat returns the expected state proof format for the given header.
-func (lc *LightChain) StateProofFormat(header types.Header) merkle.ProofFormat {
-	return lc.stateProofFormat
 }
 
 // AddStateProof adds a state proof. If it belongs to a canonical header then
 // the state range is also updated.
+// Note: it is the caller's responsibility to make sure that the proof has the
+// right format for the given application; this function only verifies the state
+// root against the corresponding header.
 func (lc *LightChain) AddStateProof(header types.Header, proof merkle.MultiProof) (err error) {
 	lc.lock.Lock()
 	defer lc.lock.Unlock()
 
-	if !merkle.IsEqual(proof.Format, lc.StateProofFormat(header)) {
-		return ErrInvalidProofFormat
-	}
 	if proof.RootHash() != header.StateRoot {
 		return ErrInvalidStateRoot
 	}
@@ -602,13 +590,8 @@ func (lc *LightChain) AddStateProof(header types.Header, proof merkle.MultiProof
 		}
 	}()
 
-	stateEnc, err := rlp.EncodeToBytes(&stateProofData{Values: proof.Values})
+	batch.Put(getStateProofKey(header.Slot, header.StateRoot), proof.Encode())
-	if err != nil {
+	lc.stateProofCache.Add(slotAndHash{header.Slot, header.StateRoot}, proof)
-		log.Error("Failed to encode state proof data", "error", err)
-		return err
-	}
-	batch.Put(getStateKey(header.Slot, header.StateRoot), stateEnc)
-	lc.stateCache.Add(slotAndHash{header.Slot, header.StateRoot}, proof.Values)
 	if !lc.IsCanonical(header) {
 		return nil
 	}

beacon/light/light_chain_test.go

@@ -230,7 +230,7 @@ func TestLightChainPrune(t *testing.T) {
 type lightChainTest struct {
 	t           *testing.T
 	db          *memorydb.Database
-	proofFormat merkle.ProofFormat
+	proofFormat merkle.CompactProofFormat
 	chain       *LightChain
 	headers     []types.Header // not added to the chain yet
 	stateProofs []testProof    // not added to the chain yet
@@ -246,10 +246,10 @@ func newLightChainTest(t *testing.T) *lightChainTest {
 	c := &lightChainTest{
 		t:           t,
 		db:          memorydb.New(),
-		proofFormat: merkle.NewIndexMapFormat().AddLeaf(42, nil).AddLeaf(67, nil),
+		proofFormat: merkle.EncodeCompactProofFormat(merkle.NewIndexMapFormat().AddLeaf(42, nil).AddLeaf(67, nil)),
 		emptyRatio:  20,
 	}
-	c.chain = NewLightChain(c.db, c.proofFormat)
+	c.chain = NewLightChain(c.db)
 	return c
 }
 
@@ -292,12 +292,12 @@ func (c *lightChainTest) checkTail(header, expTail types.Header) {
 }
 
 func (c *lightChainTest) reloadChain() {
-	c.chain = NewLightChain(c.db, c.proofFormat)
+	c.chain = NewLightChain(c.db)
 }
 
 func (c *lightChainTest) makeChain(from types.Header, targetHeadSlot uint64, addHeaders, addStateProofs bool) (tail, head types.Header) {
 	head = from
-	valueCount := merkle.ValueCount(c.proofFormat)
+	valueCount := c.proofFormat.ValueCount()
 	for head.Slot < targetHeadSlot {
 		var (
 			slot       uint64

beacon/light/sync/state_sync.go

@@ -33,7 +33,7 @@ import (
 type beaconStateServer interface {
 	request.RequestServer
 	BeaconStateTail() uint64
-	RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(*merkle.MultiProof))
+	RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.CompactProofFormat, response func(*merkle.MultiProof))
 }
 
 type StateSync struct {
@@ -41,16 +41,18 @@ type StateSync struct {
 	reqLock                       request.MultiLock
 	chain                         *light.LightChain
 	prefetch                      bool
+	syncProofFormat               merkle.CompactProofFormat
 	targetTailSlot                uint64
 	headSyncPossible              uint32
 	selfTrigger, headStateTrigger *request.ModuleTrigger
 }
 
-func NewStateSync(chain *light.LightChain, prefetch bool) *StateSync {
+func NewStateSync(chain *light.LightChain, syncProofFormat merkle.CompactProofFormat, prefetch bool) *StateSync {
 	return &StateSync{
-		chain:          chain,
+		chain:           chain,
-		prefetch:       prefetch,
+		syncProofFormat: syncProofFormat,
-		targetTailSlot: math.MaxUint64,
+		prefetch:        prefetch,
+		targetTailSlot:  math.MaxUint64,
 	}
 }
 
@@ -203,7 +205,7 @@ func (r stateRequest) CanSendTo(server *request.Server) (canSend bool, priority
 
 func (r stateRequest) SendTo(server *request.Server) {
 	reqId := r.reqLock.Send(server, r.header.StateRoot)
-	server.RequestServer.(beaconStateServer).RequestBeaconState(r.header.Slot, r.header.StateRoot, r.chain.StateProofFormat(r.header), func(proof *merkle.MultiProof) {
+	server.RequestServer.(beaconStateServer).RequestBeaconState(r.header.Slot, r.header.StateRoot, r.syncProofFormat, func(proof *merkle.MultiProof) {
 		r.lock.Lock()
 		defer r.lock.Unlock()
 

beacon/light/types/header.go

@@ -25,6 +25,8 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+var headerFormat = merkle.EncodeCompactProofFormat(merkle.NewRangeFormat(8, 15, nil))
+
 // Header defines a beacon header
 //
 // See data structure definition here:
@@ -83,7 +85,7 @@ func (bh *Header) Hash() common.Hash {
 	values[params.BhiParentRoot-8] = merkle.Value(bh.ParentRoot)
 	values[params.BhiStateRoot-8] = merkle.Value(bh.StateRoot)
 	values[params.BhiBodyRoot-8] = merkle.Value(bh.BodyRoot)
-	return merkle.MultiProof{Format: merkle.NewRangeFormat(8, 15, nil), Values: values[:]}.RootHash()
+	return merkle.MultiProof{Format: headerFormat, Values: values[:]}.RootHash()
 }
 
 // Epoch returns the epoch the header belongs to
@@ -127,7 +129,7 @@ func (bh *HeaderWithoutState) Proof(stateRoot common.Hash) merkle.MultiProof {
 	values[params.BhiParentRoot-8] = merkle.Value(bh.ParentRoot)
 	values[params.BhiStateRoot-8] = merkle.Value(stateRoot)
 	values[params.BhiBodyRoot-8] = merkle.Value(bh.BodyRoot)
-	return merkle.MultiProof{Format: merkle.NewRangeFormat(8, 15, nil), Values: values[:]}
+	return merkle.MultiProof{Format: headerFormat, Values: values[:]}
 }
 
 // FullHeader reconstructs a full Header from a HeaderWithoutState and a state root

beacon/merkle/binary_merkle.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/minio/sha256-simd"
 )
 
@@ -69,16 +68,6 @@ type ProofFormat interface {
 	Children() (left, right ProofFormat) // either both or neither should be nil
 }
 
-// IsEqual returns true if the two formats are the same
-func IsEqual(a, b ProofFormat) bool {
-	al, ar := a.Children()
-	bl, br := b.Children()
-	if al == nil || bl == nil {
-		return al == nil && bl == nil
-	}
-	return IsEqual(al, bl) && IsEqual(ar, br)
-}
-
 // ValueCount returns the number of merkle values required for this proof format
 func ValueCount(f ProofFormat) int {
 	if f == nil {
@@ -147,127 +136,6 @@ func TraverseProof(reader ProofReader, writer ProofWriter) (common.Hash, bool) {
 	return common.Hash(node), true
 }
 
-// MultiProof stores a partial Merkle tree proof
-type MultiProof struct {
-	Format ProofFormat
-	Values Values
-}
-
-// multiProofReader implements ProofReader based on a MultiProof and also allows
-// attaching further subtree readers at certain indices
-// Note: valuePtr is stored and copied as a reference because child readers read
-// from the same value list as the tree is traversed
-type multiProofReader struct {
-	format   ProofFormat              // corresponding proof format
-	values   Values                   // proof values
-	valuePtr *int                     // next index to be read from values
-	index    uint64                   // generalized tree index
-	subtrees func(uint64) ProofReader // attached subtrees
-}
-
-// children implements ProofReader
-func (mpr multiProofReader) Children() (left, right ProofReader) {
-	lf, rf := mpr.format.Children()
-	if lf == nil {
-		if mpr.subtrees != nil {
-			if subtree := mpr.subtrees(mpr.index); subtree != nil {
-				return subtree.Children()
-			}
-		}
-		return nil, nil
-	}
-	return multiProofReader{format: lf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index * 2, subtrees: mpr.subtrees},
-		multiProofReader{format: rf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index*2 + 1, subtrees: mpr.subtrees}
-}
-
-// readNode implements ProofReader
-func (mpr multiProofReader) ReadNode() (Value, bool) {
-	if l, _ := mpr.format.Children(); l == nil && len(mpr.values) > *mpr.valuePtr {
-		hash := mpr.values[*mpr.valuePtr]
-		(*mpr.valuePtr)++
-		return hash, true
-	}
-	return Value{}, false
-}
-
-// Reader creates a multiProofReader for the given proof; if subtrees != nil
-// then also attaches subtree readers at indices where the function returns a
-// non-nil reader.
-// Note that the reader can only be traversed once as the values slice is
-// sequentially consumed.
-func (mp MultiProof) Reader(subtrees func(uint64) ProofReader) multiProofReader {
-	return multiProofReader{format: mp.Format, values: mp.Values, valuePtr: new(int), index: 1, subtrees: subtrees}
-}
-
-// Finished returns true if all values have been consumed by the traversal.
-// Should be checked after TraverseProof if received from an untrusted source in
-// order to prevent DoS attacks by excess proof values.
-func (mpr multiProofReader) Finished() bool {
-	return len(mpr.values) == *mpr.valuePtr
-}
-
-// rootHash returns the root hash of the proven structure.
-func (mp MultiProof) RootHash() common.Hash {
-	reader := mp.Reader(nil)
-	hash, ok := TraverseProof(reader, nil)
-	if !ok || !reader.Finished() {
-		log.Error("MultiProof.rootHash: invalid proof format")
-	}
-	return hash
-}
-
-// multiProofWriter implements ProofWriter and creates a MultiProof with the
-// previously specified format. Also allows attaching further subtree writers at
-// certain indices.
-// Note: values is stored and copied as a reference because child writers append
-// to the same value list as the tree is traversed
-type multiProofWriter struct {
-	format   ProofFormat              // target proof format
-	values   *Values                  // target proof value list
-	index    uint64                   // generalized tree index
-	subtrees func(uint64) ProofWriter // attached subtrees
-}
-
-// NewMultiProofWriter creates a new multiproof writer with the specified format.
-// If subtrees != nil then further subtree writers are attached at indices where
-// the function returns a non-nil writer.
-// Note that the specified format should not include these attached subtrees;
-// they should be attached at leaf indices of the given format.
-// Also note that target can be nil in which case the nodes specified by the format
-// are traversed but not stored; subtree writers might still store tree data.
-func NewMultiProofWriter(format ProofFormat, target *Values, subtrees func(uint64) ProofWriter) multiProofWriter {
-	return multiProofWriter{format: format, values: target, index: 1, subtrees: subtrees}
-}
-
-// children implements ProofWriter
-func (mpw multiProofWriter) Children() (left, right ProofWriter) {
-	if mpw.subtrees != nil {
-		if subtree := mpw.subtrees(mpw.index); subtree != nil {
-			return subtree.Children()
-		}
-	}
-	lf, rf := mpw.format.Children()
-	if lf == nil {
-		return nil, nil
-	}
-	return multiProofWriter{format: lf, values: mpw.values, index: mpw.index * 2, subtrees: mpw.subtrees},
-		multiProofWriter{format: rf, values: mpw.values, index: mpw.index*2 + 1, subtrees: mpw.subtrees}
-}
-
-// writeNode implements ProofWriter
-func (mpw multiProofWriter) WriteNode(node Value) {
-	if mpw.values != nil {
-		if lf, _ := mpw.format.Children(); lf == nil {
-			*mpw.values = append(*mpw.values, node)
-		}
-	}
-	if mpw.subtrees != nil {
-		if subtree := mpw.subtrees(mpw.index); subtree != nil {
-			subtree.WriteNode(node)
-		}
-	}
-}
-
 // ProofFormatIndexMap creates a generalized tree index -> MultiProof value
 // slice index association map based on the given proof format.
 func ProofFormatIndexMap(f ProofFormat) map[uint64]int {

beacon/merkle/compact_proof.go

@@ -0,0 +1,271 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package merkle
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// CompactProofFormat is a binary compact proof format, see description here:
+// https://github.com/ChainSafe/consensus-specs/blob/feat/multiproof/ssz/merkle-proofs.md#compact-multiproofs
+type CompactProofFormat struct {
+	Format                 []byte
+	firstBit, afterLastBit int
+}
+
+// Children implements ProofFormat
+func (c CompactProofFormat) Children() (left, right ProofFormat) {
+	if bit, ok := c.readFirstBit(); !ok {
+		log.Error("Invalid compact proof format")
+	} else if bit {
+		return
+	}
+	l, r := c, c
+	if !r.skipSubtree() {
+		log.Error("Invalid compact proof format")
+	}
+	return l, r
+}
+
+// ValueCount returns the number of merkle values required for this proof
+func (c CompactProofFormat) ValueCount() int {
+	return (c.afterLastBit + 1 - c.firstBit) / 2
+}
+
+// EncodeCompactProofFormat encodes a ProofFormat into a binary compact proof format.
+func EncodeCompactProofFormat(format ProofFormat) (c CompactProofFormat) {
+	c.encodeFormatSubtree(format)
+	return
+}
+
+// readFirstBit reads the first bit of the bit vector and moves the first bit
+// pointer one bit ahead.
+func (c *CompactProofFormat) readFirstBit() (bit, ok bool) {
+	if c.firstBit >= c.afterLastBit {
+		return false, false
+	}
+	bit = c.Format[c.firstBit>>3]&(byte(128)>>(c.firstBit&7)) != 0
+	c.firstBit++
+	return bit, true
+}
+
+// skipSubtree moves the first bit pointer beyond the subtree it was pointing at
+// before and returns true if successful.
+func (c *CompactProofFormat) skipSubtree() bool {
+	if bit, ok := c.readFirstBit(); !ok {
+		return false
+	} else if bit {
+		return true
+	}
+	return c.skipSubtree() && c.skipSubtree()
+}
+
+// appendBit adds a bit at the end of the bit vector.
+func (c *CompactProofFormat) appendBit(bit bool) {
+	bytePtr := c.afterLastBit >> 3
+	if bytePtr == len(c.Format) {
+		c.Format = append(c.Format, byte(0))
+	}
+	if bit {
+		c.Format[bytePtr] += byte(128) >> (c.afterLastBit & 7)
+	}
+	c.afterLastBit++
+}
+
+// encodeFormatSubtree encodes a ProofFormat subtree at the end of the bit vector.
+func (c *CompactProofFormat) encodeFormatSubtree(format ProofFormat) {
+	if left, right := format.Children(); left == nil {
+		c.appendBit(true)
+	} else {
+		c.appendBit(false)
+		c.encodeFormatSubtree(left)
+		c.encodeFormatSubtree(right)
+	}
+}
+
+// encodeProofFormatSubtree recursively encodes a subtree of a proof format into
+// binary compact format.
+func encodeProofFormatSubtree(format ProofFormat, target *[]byte, bitLength *int) {
+	bytePtr, bitMask := *bitLength>>3, byte(128)>>(*bitLength&7)
+	*bitLength++
+	if bytePtr == len(*target) {
+		*target = append(*target, byte(0))
+	}
+	if left, right := format.Children(); left == nil {
+		(*target)[bytePtr] += bitMask
+	} else {
+		encodeProofFormatSubtree(left, target, bitLength)
+		encodeProofFormatSubtree(right, target, bitLength)
+	}
+}
+
+// MultiProof stores a partial Merkle tree proof
+type MultiProof struct {
+	Format CompactProofFormat
+	Values Values
+}
+
+// Encode encodes a MultiProof into a byte slice
+func (m *MultiProof) Encode() []byte {
+	lf := len(m.Format.Format)
+	enc := make([]byte, lf+32*len(m.Values))
+	copy(enc[:lf], m.Format.Format)
+	for i, value := range m.Values {
+		copy(enc[lf+i*32:lf+(i+1)*32], value[:])
+	}
+	return enc
+}
+
+// Decode decodes a MultiProof from a byte slice
+func (m *MultiProof) Decode(enc []byte) error {
+	valueCount := len(enc) * 4 / 129
+	lf := (valueCount + 3) / 4
+	if len(enc) != lf+32*valueCount {
+		return errors.New("Invalid length for encoded MultiProof")
+	}
+	format := CompactProofFormat{
+		Format:       make([]byte, lf),
+		afterLastBit: valueCount*2 - 1,
+	}
+	copy(format.Format, enc[:lf])
+	if f := format; !f.skipSubtree() || f.firstBit != f.afterLastBit {
+		log.Error("Invalid compact proof format")
+	}
+	m.Format, m.Values = format, make(Values, valueCount)
+	for i := range m.Values {
+		copy(m.Values[i][:], enc[lf+i*32:lf+(i+1)*32])
+	}
+	return nil
+}
+
+// multiProofReader implements ProofReader based on a MultiProof and also allows
+// attaching further subtree readers at certain indices
+// Note: valuePtr is stored and copied as a reference because child readers read
+// from the same value list as the tree is traversed
+type multiProofReader struct {
+	format   ProofFormat              // corresponding proof format
+	values   Values                   // proof values
+	valuePtr *int                     // next index to be read from values
+	index    uint64                   // generalized tree index
+	subtrees func(uint64) ProofReader // attached subtrees
+}
+
+// children implements ProofReader
+func (mpr multiProofReader) Children() (left, right ProofReader) {
+	lf, rf := mpr.format.Children()
+	if lf == nil {
+		if mpr.subtrees != nil {
+			if subtree := mpr.subtrees(mpr.index); subtree != nil {
+				return subtree.Children()
+			}
+		}
+		return nil, nil
+	}
+	return multiProofReader{format: lf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index * 2, subtrees: mpr.subtrees},
+		multiProofReader{format: rf, values: mpr.values, valuePtr: mpr.valuePtr, index: mpr.index*2 + 1, subtrees: mpr.subtrees}
+}
+
+// readNode implements ProofReader
+func (mpr multiProofReader) ReadNode() (Value, bool) {
+	if l, _ := mpr.format.Children(); l == nil && len(mpr.values) > *mpr.valuePtr {
+		hash := mpr.values[*mpr.valuePtr]
+		(*mpr.valuePtr)++
+		return hash, true
+	}
+	return Value{}, false
+}
+
+// Reader creates a multiProofReader for the given proof; if subtrees != nil
+// then also attaches subtree readers at indices where the function returns a
+// non-nil reader.
+// Note that the reader can only be traversed once as the values slice is
+// sequentially consumed.
+func (mp MultiProof) Reader(subtrees func(uint64) ProofReader) multiProofReader {
+	return multiProofReader{format: mp.Format, values: mp.Values, valuePtr: new(int), index: 1, subtrees: subtrees}
+}
+
+// Finished returns true if all values have been consumed by the traversal.
+// Should be checked after TraverseProof if received from an untrusted source in
+// order to prevent DoS attacks by excess proof values.
+func (mpr multiProofReader) Finished() bool {
+	return len(mpr.values) == *mpr.valuePtr
+}
+
+// rootHash returns the root hash of the proven structure.
+func (mp MultiProof) RootHash() common.Hash {
+	reader := mp.Reader(nil)
+	hash, ok := TraverseProof(reader, nil)
+	if !ok || !reader.Finished() {
+		log.Error("MultiProof.rootHash: invalid proof format")
+	}
+	return hash
+}
+
+// multiProofWriter implements ProofWriter and creates a MultiProof with the
+// previously specified format. Also allows attaching further subtree writers at
+// certain indices.
+// Note: values is stored and copied as a reference because child writers append
+// to the same value list as the tree is traversed
+type multiProofWriter struct {
+	format   ProofFormat              // target proof format
+	values   *Values                  // target proof value list
+	index    uint64                   // generalized tree index
+	subtrees func(uint64) ProofWriter // attached subtrees
+}
+
+// NewMultiProofWriter creates a new multiproof writer with the specified format.
+// If subtrees != nil then further subtree writers are attached at indices where
+// the function returns a non-nil writer.
+// Note that the specified format should not include these attached subtrees;
+// they should be attached at leaf indices of the given format.
+// Also note that target can be nil in which case the nodes specified by the format
+// are traversed but not stored; subtree writers might still store tree data.
+func NewMultiProofWriter(format ProofFormat, target *Values, subtrees func(uint64) ProofWriter) multiProofWriter {
+	return multiProofWriter{format: format, values: target, index: 1, subtrees: subtrees}
+}
+
+// children implements ProofWriter
+func (mpw multiProofWriter) Children() (left, right ProofWriter) {
+	if mpw.subtrees != nil {
+		if subtree := mpw.subtrees(mpw.index); subtree != nil {
+			return subtree.Children()
+		}
+	}
+	lf, rf := mpw.format.Children()
+	if lf == nil {
+		return nil, nil
+	}
+	return multiProofWriter{format: lf, values: mpw.values, index: mpw.index * 2, subtrees: mpw.subtrees},
+		multiProofWriter{format: rf, values: mpw.values, index: mpw.index*2 + 1, subtrees: mpw.subtrees}
+}
+
+// writeNode implements ProofWriter
+func (mpw multiProofWriter) WriteNode(node Value) {
+	if mpw.values != nil {
+		if lf, _ := mpw.format.Children(); lf == nil {
+			*mpw.values = append(*mpw.values, node)
+		}
+	}
+	if mpw.subtrees != nil {
+		if subtree := mpw.subtrees(mpw.index); subtree != nil {
+			subtree.WriteNode(node)
+		}
+	}
+}

cmd/blsync/block_sync.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/beacon/light/request"
 	lsync "github.com/ethereum/go-ethereum/beacon/light/sync"
 	"github.com/ethereum/go-ethereum/beacon/light/types"
-	"github.com/ethereum/go-ethereum/beacon/merkle"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
 	ctypes "github.com/ethereum/go-ethereum/core/types"
@@ -46,12 +45,6 @@ type beaconBlockServer interface {
 	RequestBeaconBlock(blockRoot common.Hash, response func(*capella.BeaconBlock))
 }
 
-type beaconStateServer interface {
-	request.RequestServer
-	BeaconStateTail() uint64
-	RequestBeaconState(slot uint64, stateRoot common.Hash, format merkle.ProofFormat, response func(merkle.MultiProof))
-}
-
 type beaconBlockSync struct {
 	lock         sync.Mutex
 	reqLock      request.MultiLock

cmd/blsync/main.go

@@ -70,16 +70,16 @@ func main() {
 }
 
 var (
-	stateProofFormat    merkle.ProofFormat // requested multiproof format
+	stateProofFormat    merkle.CompactProofFormat // requested multiproof format
-	execBlockIndex      int                // index of execution block root in proof.Values where proof.Format == stateProofFormat
+	execBlockIndex      int                       // index of execution block root in proof.Values where proof.Format == stateProofFormat
-	finalizedBlockIndex int                // index of finalized block root in proof.Values where proof.Format == stateProofFormat
+	finalizedBlockIndex int                       // index of finalized block root in proof.Values where proof.Format == stateProofFormat
 )
 
 func blsync(ctx *cli.Context) error {
 	if !ctx.IsSet(utils.BeaconApiFlag.Name) {
 		utils.Fatalf("Beacon node light client API URL not specified")
 	}
-	stateProofFormat = merkle.NewIndexMapFormat().AddLeaf(params.BsiExecHead, nil).AddLeaf(params.BsiFinalBlock, nil)
+	stateProofFormat = merkle.EncodeCompactProofFormat(merkle.NewIndexMapFormat().AddLeaf(params.BsiExecHead, nil).AddLeaf(params.BsiFinalBlock, nil))
 	var (
 		stateIndexMap = merkle.ProofFormatIndexMap(stateProofFormat)
 		chainConfig   = makeChainConfig(ctx)
@@ -103,7 +103,7 @@ func blsync(ctx *cli.Context) error {
 		committeeChain  = light.NewCommitteeChain(db, chainConfig.Forks, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name), light.BLSVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() })
 		checkpointStore = light.NewCheckpointStore(db, committeeChain)
 		headValidator   = light.NewHeadValidator(committeeChain)
-		lightChain      = light.NewLightChain(db, stateProofFormat)
+		lightChain      = light.NewLightChain(db)
 	)
 	committeeChain.SetGenesisData(chainConfig.GenesisData)
 	headUpdater := sync.NewHeadUpdater(headValidator, committeeChain)
@@ -116,7 +116,7 @@ func blsync(ctx *cli.Context) error {
 	checkpointInit := sync.NewCheckpointInit(committeeChain, checkpointStore, chainConfig.Checkpoint)
 	forwardSync := sync.NewForwardUpdateSync(committeeChain)
 	headerSync := sync.NewHeaderSync(lightChain, false)
-	stateSync := sync.NewStateSync(lightChain, true)
+	stateSync := sync.NewStateSync(lightChain, stateProofFormat, true)
 	beaconBlockSync := newBeaconBlockSyncer(lightChain)
 	engineApiUpdater := &engineApiUpdater{ //TODO constructor
 		client:     makeRPCClient(ctx),