les, light, trie: iterator based proofs, no rlp types

les/handler.go

@@ -710,7 +710,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 		// A batch of merkle proofs arrived to one of our previous requests
 		var resp struct {
 			ReqID, BV uint64
-			Data      [][]rlp.RawValue
+			Data      [][][]byte
 		}
 		if err := msg.Decode(&resp); err != nil {
 			return errResp(ErrDecode, "msg %v: %v", msg, err)

les/handler_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
@@ -316,7 +315,7 @@ func testGetProofs(t *testing.T, protocol int) {
 	defer peer.close()
 
 	var proofreqs []ProofReq
-	var proofs [][]rlp.RawValue
+	var proofs [][][]byte
 
 	accounts := []common.Address{testBankAddress, acc1Addr, acc2Addr, {}}
 	for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ {

les/odr_requests.go

@@ -215,7 +215,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error {
 	if msg.MsgType != MsgProofs {
 		return errInvalidMessageType
 	}
-	proofs := msg.Obj.([][]rlp.RawValue)
+	proofs := msg.Obj.([][][]byte)
 	if len(proofs) != 1 {
 		return errMultipleEntries
 	}
@@ -286,7 +286,7 @@ type ChtReq struct {
 
 type ChtResp struct {
 	Header *types.Header
-	Proof  []rlp.RawValue
+	Proof  [][]byte
 }
 
 // ODR request type for requesting headers by Canonical Hash Trie, see LesOdrRequest interface

les/protocol.go

@@ -168,4 +168,4 @@ type CodeData []struct {
 	Value []byte
 }
 
-type proofsData [][]rlp.RawValue
+type proofsData [][][]byte

light/odr.go

@@ -27,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // NoOdr is the default context passed to an ODR capable function when the ODR
@@ -80,7 +79,7 @@ type TrieRequest struct {
 	OdrRequest
 	Id    *TrieID
 	Key   []byte
-	Proof []rlp.RawValue
+	Proof [][]byte
 }
 
 // StoreResult stores the retrieved data in local database
@@ -89,7 +88,7 @@ func (req *TrieRequest) StoreResult(db ethdb.Database) {
 }
 
 // storeProof stores the new trie nodes obtained from a merkle proof in the database
-func storeProof(db ethdb.Database, proof []rlp.RawValue) {
+func storeProof(db ethdb.Database, proof [][]byte) {
 	for _, buf := range proof {
 		hash := crypto.Keccak256(buf)
 		val, _ := db.Get(hash)
@@ -145,7 +144,7 @@ type ChtRequest struct {
 	ChtRoot          common.Hash
 	Header           *types.Header
 	Td               *big.Int
-	Proof            []rlp.RawValue
+	Proof            [][]byte
 }
 
 // StoreResult stores the retrieved data in local database

trie/iterator.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // Iterator is a key-value trie iterator that traverses a Trie.
@@ -55,31 +56,50 @@ func (it *Iterator) Next() bool {
 	return false
 }
 
+// Prove generates the Merkle proof for the leaf node the iterator is currently
+// positioned on.
+func (it *Iterator) Prove() [][]byte {
+	return it.nodeIt.LeafProof()
+}
+
 // NodeIterator is an iterator to traverse the trie pre-order.
 type NodeIterator interface {
 	// Next moves the iterator to the next node. If the parameter is false, any child
 	// nodes will be skipped.
 	Next(bool) bool
+
 	// Error returns the error status of the iterator.
 	Error() error
 
 	// Hash returns the hash of the current node.
 	Hash() common.Hash
+
 	// Parent returns the hash of the parent of the current node. The hash may be the one
 	// grandparent if the immediate parent is an internal node with no hash.
 	Parent() common.Hash
+
 	// Path returns the hex-encoded path to the current node.
 	// Callers must not retain references to the return value after calling Next.
 	// For leaf nodes, the last element of the path is the 'terminator symbol' 0x10.
 	Path() []byte
 
 	// Leaf returns true iff the current node is a leaf node.
-	// LeafBlob, LeafKey return the contents and key of the leaf node. These
-	// method panic if the iterator is not positioned at a leaf.
-	// Callers must not retain references to their return value after calling Next
 	Leaf() bool
-	LeafBlob() []byte
+
+	// LeafKey returns the key of the leaf. The method panics if the iterator is not
+	// positioned at a leaf. Callers must not retain references to the value after
+	// calling Next.
 	LeafKey() []byte
+
+	// LeafBlob returns the content of the leaf. The method panics if the iterator
+	// is not positioned at a leaf. Callers must not retain references to the value
+	// after calling Next.
+	LeafBlob() []byte
+
+	// LeafProof returns the Merkle proof of the leaf. The method panics if the
+	// iterator is not positioned at a leaf. Callers must not retain references
+	// to the value after calling Next.
+	LeafProof() [][]byte
 }
 
 // nodeIteratorState represents the iteration state at one particular node of the
@@ -139,6 +159,15 @@ func (it *nodeIterator) Leaf() bool {
 	return hasTerm(it.path)
 }
 
+func (it *nodeIterator) LeafKey() []byte {
+	if len(it.stack) > 0 {
+		if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
+			return hexToKeybytes(it.path)
+		}
+	}
+	panic("not at leaf")
+}
+
 func (it *nodeIterator) LeafBlob() []byte {
 	if len(it.stack) > 0 {
 		if node, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
@@ -148,10 +177,22 @@ func (it *nodeIterator) LeafBlob() []byte {
 	panic("not at leaf")
 }
 
-func (it *nodeIterator) LeafKey() []byte {
+func (it *nodeIterator) LeafProof() [][]byte {
 	if len(it.stack) > 0 {
 		if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok {
-			return hexToKeybytes(it.path)
+			hasher := newHasher(0, 0)
+			proofs := make([][]byte, 0, len(it.stack))
+
+			for i, item := range it.stack[:len(it.stack)-1] {
+				// Gather nodes that end up as hash nodes (or the root)
+				node, _, _ := hasher.hashChildren(item.node, nil)
+				hashed, _ := hasher.store(node, nil, false)
+				if _, ok := hashed.(hashNode); ok || i == 0 {
+					enc, _ := rlp.EncodeToBytes(node)
+					proofs = append(proofs, enc)
+				}
+			}
+			return proofs
 		}
 	}
 	panic("not at leaf")
@@ -361,12 +402,16 @@ func (it *differenceIterator) Leaf() bool {
 	return it.b.Leaf()
 }
 
+func (it *differenceIterator) LeafKey() []byte {
+	return it.b.LeafKey()
+}
+
 func (it *differenceIterator) LeafBlob() []byte {
 	return it.b.LeafBlob()
 }
 
-func (it *differenceIterator) LeafKey() []byte {
+func (it *differenceIterator) LeafProof() [][]byte {
-	return it.b.LeafKey()
+	return it.b.LeafProof()
 }
 
 func (it *differenceIterator) Path() []byte {
@@ -464,12 +509,16 @@ func (it *unionIterator) Leaf() bool {
 	return (*it.items)[0].Leaf()
 }
 
+func (it *unionIterator) LeafKey() []byte {
+	return (*it.items)[0].LeafKey()
+}
+
 func (it *unionIterator) LeafBlob() []byte {
 	return (*it.items)[0].LeafBlob()
 }
 
-func (it *unionIterator) LeafKey() []byte {
+func (it *unionIterator) LeafProof() [][]byte {
-	return (*it.items)[0].LeafKey()
+	return (*it.items)[0].LeafProof()
 }
 
 func (it *unionIterator) Path() []byte {

trie/proof.go

@@ -36,7 +36,7 @@ import (
 // contains all nodes of the longest existing prefix of the key
 // (at least the root node), ending with the node that proves the
 // absence of the key.
-func (t *Trie) Prove(key []byte) []rlp.RawValue {
+func (t *Trie) Prove(key []byte) [][]byte {
 	// Collect all nodes on the path to key.
 	key = keybytesToHex(key)
 	nodes := []node{}
@@ -68,7 +68,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
 		}
 	}
 	hasher := newHasher(0, 0)
-	proof := make([]rlp.RawValue, 0, len(nodes))
+	proof := make([][]byte, 0, len(nodes))
 	for i, n := range nodes {
 		// Don't bother checking for errors here since hasher panics
 		// if encoding doesn't work and we're not writing to any database.
@@ -88,7 +88,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
 // value for key in a trie with the given root hash. VerifyProof
 // returns an error if the proof contains invalid trie nodes or the
 // wrong value.
-func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value []byte, err error) {
+func VerifyProof(rootHash common.Hash, key []byte, proof [][]byte) (value []byte, err error) {
 	key = keybytesToHex(key)
 	sha := sha3.NewKeccak256()
 	wantHash := rootHash.Bytes()
@@ -107,10 +107,8 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value
 		case nil:
 			if i != len(proof)-1 {
 				return nil, fmt.Errorf("key mismatch at proof node %d", i)
-			} else {
-				// The trie doesn't contain the key.
-				return nil, nil
 			}
+			return nil, nil // The trie doesn't contain the key.
 		case hashNode:
 			key = keyrest
 			wantHash = cld

trie/proof_test.go

@@ -24,27 +24,47 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
 func init() {
 	mrand.Seed(time.Now().Unix())
 }
 
+// makeProvers creates Merkle trie provers based on different implementations to
+// test all variations.
+func makeProvers(trie *Trie) []func(key []byte) [][]byte {
+	var provers []func(key []byte) [][]byte
+
+	// Create a direct trie based Merkle prover
+	provers = append(provers, func(key []byte) [][]byte {
+		return trie.Prove(key)
+	})
+	// Create a leaf iterator based Merkle prover
+	provers = append(provers, func(key []byte) [][]byte {
+		if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) {
+			return it.Prove()
+		}
+		return nil
+	})
+	return provers
+}
+
 func TestProof(t *testing.T) {
 	trie, vals := randomTrie(500)
 	root := trie.Hash()
-	for _, kv := range vals {
+	for i, prover := range makeProvers(trie) {
-		proof := trie.Prove(kv.k)
+		for _, kv := range vals {
-		if proof == nil {
+			proof := prover(kv.k)
-			t.Fatalf("missing key %x while constructing proof", kv.k)
+			if proof == nil {
-		}
+				t.Fatalf("prover %d: missing key %x while constructing proof", i, kv.k)
-		val, err := VerifyProof(root, kv.k, proof)
+			}
-		if err != nil {
+			val, err := VerifyProof(root, kv.k, proof)
-			t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %x", kv.k, err, proof)
+			if err != nil {
-		}
+				t.Fatalf("prover %d: failed to verify proof for key %x: %v\nraw proof: %x", i, kv.k, err, proof)
-		if !bytes.Equal(val, kv.v) {
+			}
-			t.Fatalf("VerifyProof returned wrong value for key %x: got %x, want %x", kv.k, val, kv.v)
+			if !bytes.Equal(val, kv.v) {
+				t.Fatalf("prover %d: verified valuemismatch for key %x: have %x, want %x", i, kv.k, val, kv.v)
+			}
 		}
 	}
 }
@@ -52,33 +72,61 @@ func TestProof(t *testing.T) {
 func TestOneElementProof(t *testing.T) {
 	trie := new(Trie)
 	updateString(trie, "k", "v")
-	proof := trie.Prove([]byte("k"))
+	for i, prover := range makeProvers(trie) {
-	if proof == nil {
+		proof := prover([]byte("k"))
-		t.Fatal("nil proof")
+		if proof == nil {
-	}
+			t.Fatalf("prover %d: nil proof", i)
-	if len(proof) != 1 {
+		}
-		t.Error("proof should have one element")
+		if len(proof) != 1 {
-	}
+			t.Errorf("prover %d: proof should have one element", i)
-	val, err := VerifyProof(trie.Hash(), []byte("k"), proof)
+		}
-	if err != nil {
+		val, err := VerifyProof(trie.Hash(), []byte("k"), proof)
-		t.Fatalf("VerifyProof error: %v\nraw proof: %x", err, proof)
+		if err != nil {
-	}
+			t.Fatalf("prover %d: failed to verify proof: %v\nraw proof: %x", i, err, proof)
-	if !bytes.Equal(val, []byte("v")) {
+		}
-		t.Fatalf("VerifyProof returned wrong value: got %x, want 'k'", val)
+		if !bytes.Equal(val, []byte("v")) {
+			t.Fatalf("prover %d: verified valuemismatch: have %x, want 'k'", i, val)
+		}
 	}
 }
 
-func TestVerifyBadProof(t *testing.T) {
+func TestBadProof(t *testing.T) {
 	trie, vals := randomTrie(800)
 	root := trie.Hash()
-	for _, kv := range vals {
+	for i, prover := range makeProvers(trie) {
-		proof := trie.Prove(kv.k)
+		for _, kv := range vals {
-		if proof == nil {
+			proof := prover(kv.k)
-			t.Fatal("nil proof")
+			if proof == nil {
+				t.Fatalf("prover %d: nil proof", i)
+			}
+			mutateByte(proof[mrand.Intn(len(proof))])
+			if _, err := VerifyProof(root, kv.k, proof); err == nil {
+				t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k)
+			}
 		}
-		mutateByte(proof[mrand.Intn(len(proof))])
+	}
-		if _, err := VerifyProof(root, kv.k, proof); err == nil {
+}
-			t.Fatalf("expected proof to fail for key %x", kv.k)
+
+// Tests that missing keys can also be proven. The test explicitly uses a single
+// entry trie and checks for missing keys both before and after the single entry.
+func TestMissingKeyProof(t *testing.T) {
+	trie := new(Trie)
+	updateString(trie, "k", "v")
+
+	for i, key := range []string{"a", "j", "l", "z"} {
+		proof := trie.Prove([]byte(key))
+		if proof == nil {
+			t.Fatalf("test %d: nil proof", i)
+		}
+		if len(proof) != 1 {
+			t.Errorf("test %d: proof should have one element", i)
+		}
+		val, err := VerifyProof(trie.Hash(), []byte(key), proof)
+		if err != nil {
+			t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof)
+		}
+		if val != nil {
+			t.Fatalf("test %d: verified valuemismatch: have %x, want nil", i, val)
 		}
 	}
 }
@@ -114,7 +162,7 @@ func BenchmarkVerifyProof(b *testing.B) {
 	trie, vals := randomTrie(100)
 	root := trie.Hash()
 	var keys []string
-	var proofs [][]rlp.RawValue
+	var proofs [][][]byte
 	for k := range vals {
 		keys = append(keys, k)
 		proofs = append(proofs, trie.Prove([]byte(k)))

trie/sync.go

@@ -45,11 +45,15 @@ type request struct {
 	callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
 }
 
-// SyncResult is a simple list to return missing nodes along with their request
+// SyncResult represents a response to a trie node retrieval request. The result
-// hashes.
+// data might be a simple binary blob if returning only a single node, or it may
+// be a batch of trie leaves (with associated merkle proofs) if returning batched
+// results.
 type SyncResult struct {
-	Hash common.Hash // Hash of the originally unknown trie node
+	Hash   common.Hash // Hash of the originally unknown trie node
-	Data []byte      // Data content of the retrieved node
+	Data   []byte      // Data content of the retrieved node, in node-sync mode
+	Leaves [][]byte    // Trie leaves rooted under the specified hash, in leaf-sync mode
+	Proofs [][]byte    // Proofs to validate the leaves, in leaf-sync mode, if leaves are partial
 }
 
 // syncMemBatch is an in-memory buffer of successfully downloaded but not yet

trie/sync_test.go

@@ -120,7 +120,7 @@ func testIterativeTrieSync(t *testing.T, batch int) {
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
-			results[i] = SyncResult{hash, data}
+			results[i] = SyncResult{Hash: hash, Data: data}
 		}
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
@@ -153,7 +153,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) {
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
-			results[i] = SyncResult{hash, data}
+			results[i] = SyncResult{Hash: hash, Data: data}
 		}
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
@@ -193,7 +193,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) {
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
-			results = append(results, SyncResult{hash, data})
+			results = append(results, SyncResult{Hash: hash, Data: data})
 		}
 		// Feed the retrieved results back and queue new tasks
 		if _, index, err := sched.Process(results); err != nil {
@@ -233,7 +233,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) {
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
-			results = append(results, SyncResult{hash, data})
+			results = append(results, SyncResult{Hash: hash, Data: data})
 
 			if len(results) >= cap(results) {
 				break
@@ -282,7 +282,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) {
 			}
 			requested[hash] = struct{}{}
 
-			results[i] = SyncResult{hash, data}
+			results[i] = SyncResult{Hash: hash, Data: data}
 		}
 		if _, index, err := sched.Process(results); err != nil {
 			t.Fatalf("failed to process result #%d: %v", index, err)
@@ -316,7 +316,7 @@ func TestIncompleteTrieSync(t *testing.T) {
 			if err != nil {
 				t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
 			}
-			results[i] = SyncResult{hash, data}
+			results[i] = SyncResult{Hash: hash, Data: data}
 		}
 		// Process each of the trie nodes
 		if _, index, err := sched.Process(results); err != nil {