diff --git a/les/handler.go b/les/handler.go index df7eb6af51..490e51167e 100644 --- a/les/handler.go +++ b/les/handler.go @@ -710,7 +710,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // A batch of merkle proofs arrived to one of our previous requests var resp struct { ReqID, BV uint64 - Data [][]rlp.RawValue + Data [][][]byte } if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) diff --git a/les/handler_test.go b/les/handler_test.go index b1f1aa0951..93e2763026 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -316,7 +315,7 @@ func testGetProofs(t *testing.T, protocol int) { defer peer.close() var proofreqs []ProofReq - var proofs [][]rlp.RawValue + var proofs [][][]byte accounts := []common.Address{testBankAddress, acc1Addr, acc2Addr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { diff --git a/les/odr_requests.go b/les/odr_requests.go index 1f853b3417..3f322da660 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -215,7 +215,7 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { if msg.MsgType != MsgProofs { return errInvalidMessageType } - proofs := msg.Obj.([][]rlp.RawValue) + proofs := msg.Obj.([][][]byte) if len(proofs) != 1 { return errMultipleEntries } @@ -286,7 +286,7 @@ type ChtReq struct { type ChtResp struct { Header *types.Header - Proof []rlp.RawValue + Proof [][]byte } // ODR request type for requesting headers by Canonical Hash Trie, see LesOdrRequest interface diff --git a/les/protocol.go b/les/protocol.go index 33d930ee0c..677feee818 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -168,4 +168,4 @@ type CodeData []struct { Value []byte } -type proofsData [][]rlp.RawValue +type proofsData [][][]byte diff --git a/light/odr.go b/light/odr.go index d19a488f62..2ed654ecd4 100644 --- a/light/odr.go +++ b/light/odr.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" ) // NoOdr is the default context passed to an ODR capable function when the ODR @@ -80,7 +79,7 @@ type TrieRequest struct { OdrRequest Id *TrieID Key []byte - Proof []rlp.RawValue + Proof [][]byte } // StoreResult stores the retrieved data in local database @@ -89,7 +88,7 @@ func (req *TrieRequest) StoreResult(db ethdb.Database) { } // storeProof stores the new trie nodes obtained from a merkle proof in the database -func storeProof(db ethdb.Database, proof []rlp.RawValue) { +func storeProof(db ethdb.Database, proof [][]byte) { for _, buf := range proof { hash := crypto.Keccak256(buf) val, _ := db.Get(hash) @@ -145,7 +144,7 @@ type ChtRequest struct { ChtRoot common.Hash Header *types.Header Td *big.Int - Proof []rlp.RawValue + Proof [][]byte } // StoreResult stores the retrieved data in local database diff --git a/trie/iterator.go b/trie/iterator.go index 76146c0d64..74e936a016 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" ) // Iterator is a key-value trie iterator that traverses a Trie. @@ -55,31 +56,50 @@ func (it *Iterator) Next() bool { return false } +// Prove generates the Merkle proof for the leaf node the iterator is currently +// positioned on. +func (it *Iterator) Prove() [][]byte { + return it.nodeIt.LeafProof() +} + // NodeIterator is an iterator to traverse the trie pre-order. type NodeIterator interface { // Next moves the iterator to the next node. If the parameter is false, any child // nodes will be skipped. Next(bool) bool + // Error returns the error status of the iterator. Error() error // Hash returns the hash of the current node. Hash() common.Hash + // Parent returns the hash of the parent of the current node. The hash may be the one // grandparent if the immediate parent is an internal node with no hash. Parent() common.Hash + // Path returns the hex-encoded path to the current node. // Callers must not retain references to the return value after calling Next. // For leaf nodes, the last element of the path is the 'terminator symbol' 0x10. Path() []byte // Leaf returns true iff the current node is a leaf node. - // LeafBlob, LeafKey return the contents and key of the leaf node. These - // method panic if the iterator is not positioned at a leaf. - // Callers must not retain references to their return value after calling Next Leaf() bool - LeafBlob() []byte + + // LeafKey returns the key of the leaf. The method panics if the iterator is not + // positioned at a leaf. Callers must not retain references to the value after + // calling Next. LeafKey() []byte + + // LeafBlob returns the content of the leaf. The method panics if the iterator + // is not positioned at a leaf. Callers must not retain references to the value + // after calling Next. + LeafBlob() []byte + + // LeafProof returns the Merkle proof of the leaf. The method panics if the + // iterator is not positioned at a leaf. Callers must not retain references + // to the value after calling Next. + LeafProof() [][]byte } // nodeIteratorState represents the iteration state at one particular node of the @@ -139,6 +159,15 @@ func (it *nodeIterator) Leaf() bool { return hasTerm(it.path) } +func (it *nodeIterator) LeafKey() []byte { + if len(it.stack) > 0 { + if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { + return hexToKeybytes(it.path) + } + } + panic("not at leaf") +} + func (it *nodeIterator) LeafBlob() []byte { if len(it.stack) > 0 { if node, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { @@ -148,10 +177,22 @@ func (it *nodeIterator) LeafBlob() []byte { panic("not at leaf") } -func (it *nodeIterator) LeafKey() []byte { +func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - return hexToKeybytes(it.path) + hasher := newHasher(0, 0) + proofs := make([][]byte, 0, len(it.stack)) + + for i, item := range it.stack[:len(it.stack)-1] { + // Gather nodes that end up as hash nodes (or the root) + node, _, _ := hasher.hashChildren(item.node, nil) + hashed, _ := hasher.store(node, nil, false) + if _, ok := hashed.(hashNode); ok || i == 0 { + enc, _ := rlp.EncodeToBytes(node) + proofs = append(proofs, enc) + } + } + return proofs } } panic("not at leaf") @@ -361,12 +402,16 @@ func (it *differenceIterator) Leaf() bool { return it.b.Leaf() } +func (it *differenceIterator) LeafKey() []byte { + return it.b.LeafKey() +} + func (it *differenceIterator) LeafBlob() []byte { return it.b.LeafBlob() } -func (it *differenceIterator) LeafKey() []byte { - return it.b.LeafKey() +func (it *differenceIterator) LeafProof() [][]byte { + return it.b.LeafProof() } func (it *differenceIterator) Path() []byte { @@ -464,12 +509,16 @@ func (it *unionIterator) Leaf() bool { return (*it.items)[0].Leaf() } +func (it *unionIterator) LeafKey() []byte { + return (*it.items)[0].LeafKey() +} + func (it *unionIterator) LeafBlob() []byte { return (*it.items)[0].LeafBlob() } -func (it *unionIterator) LeafKey() []byte { - return (*it.items)[0].LeafKey() +func (it *unionIterator) LeafProof() [][]byte { + return (*it.items)[0].LeafProof() } func (it *unionIterator) Path() []byte { diff --git a/trie/proof.go b/trie/proof.go index 298f648c4b..3d8ba80e45 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -36,7 +36,7 @@ import ( // contains all nodes of the longest existing prefix of the key // (at least the root node), ending with the node that proves the // absence of the key. -func (t *Trie) Prove(key []byte) []rlp.RawValue { +func (t *Trie) Prove(key []byte) [][]byte { // Collect all nodes on the path to key. key = keybytesToHex(key) nodes := []node{} @@ -68,7 +68,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { } } hasher := newHasher(0, 0) - proof := make([]rlp.RawValue, 0, len(nodes)) + proof := make([][]byte, 0, len(nodes)) for i, n := range nodes { // Don't bother checking for errors here since hasher panics // if encoding doesn't work and we're not writing to any database. @@ -88,7 +88,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { // value for key in a trie with the given root hash. VerifyProof // returns an error if the proof contains invalid trie nodes or the // wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value []byte, err error) { +func VerifyProof(rootHash common.Hash, key []byte, proof [][]byte) (value []byte, err error) { key = keybytesToHex(key) sha := sha3.NewKeccak256() wantHash := rootHash.Bytes() @@ -107,10 +107,8 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value case nil: if i != len(proof)-1 { return nil, fmt.Errorf("key mismatch at proof node %d", i) - } else { - // The trie doesn't contain the key. - return nil, nil } + return nil, nil // The trie doesn't contain the key. case hashNode: key = keyrest wantHash = cld diff --git a/trie/proof_test.go b/trie/proof_test.go index 91ebcd4a57..647dfeb0cb 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -24,27 +24,47 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" ) func init() { mrand.Seed(time.Now().Unix()) } +// makeProvers creates Merkle trie provers based on different implementations to +// test all variations. +func makeProvers(trie *Trie) []func(key []byte) [][]byte { + var provers []func(key []byte) [][]byte + + // Create a direct trie based Merkle prover + provers = append(provers, func(key []byte) [][]byte { + return trie.Prove(key) + }) + // Create a leaf iterator based Merkle prover + provers = append(provers, func(key []byte) [][]byte { + if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { + return it.Prove() + } + return nil + }) + return provers +} + func TestProof(t *testing.T) { trie, vals := randomTrie(500) root := trie.Hash() - for _, kv := range vals { - proof := trie.Prove(kv.k) - if proof == nil { - t.Fatalf("missing key %x while constructing proof", kv.k) - } - val, err := VerifyProof(root, kv.k, proof) - if err != nil { - t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %x", kv.k, err, proof) - } - if !bytes.Equal(val, kv.v) { - t.Fatalf("VerifyProof returned wrong value for key %x: got %x, want %x", kv.k, val, kv.v) + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: missing key %x while constructing proof", i, kv.k) + } + val, err := VerifyProof(root, kv.k, proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof for key %x: %v\nraw proof: %x", i, kv.k, err, proof) + } + if !bytes.Equal(val, kv.v) { + t.Fatalf("prover %d: verified valuemismatch for key %x: have %x, want %x", i, kv.k, val, kv.v) + } } } } @@ -52,33 +72,61 @@ func TestProof(t *testing.T) { func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") - proof := trie.Prove([]byte("k")) - if proof == nil { - t.Fatal("nil proof") - } - if len(proof) != 1 { - t.Error("proof should have one element") - } - val, err := VerifyProof(trie.Hash(), []byte("k"), proof) - if err != nil { - t.Fatalf("VerifyProof error: %v\nraw proof: %x", err, proof) - } - if !bytes.Equal(val, []byte("v")) { - t.Fatalf("VerifyProof returned wrong value: got %x, want 'k'", val) + for i, prover := range makeProvers(trie) { + proof := prover([]byte("k")) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + if len(proof) != 1 { + t.Errorf("prover %d: proof should have one element", i) + } + val, err := VerifyProof(trie.Hash(), []byte("k"), proof) + if err != nil { + t.Fatalf("prover %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if !bytes.Equal(val, []byte("v")) { + t.Fatalf("prover %d: verified valuemismatch: have %x, want 'k'", i, val) + } } } -func TestVerifyBadProof(t *testing.T) { +func TestBadProof(t *testing.T) { trie, vals := randomTrie(800) root := trie.Hash() - for _, kv := range vals { - proof := trie.Prove(kv.k) - if proof == nil { - t.Fatal("nil proof") + for i, prover := range makeProvers(trie) { + for _, kv := range vals { + proof := prover(kv.k) + if proof == nil { + t.Fatalf("prover %d: nil proof", i) + } + mutateByte(proof[mrand.Intn(len(proof))]) + if _, err := VerifyProof(root, kv.k, proof); err == nil { + t.Fatalf("prover %d: expected proof to fail for key %x", i, kv.k) + } } - mutateByte(proof[mrand.Intn(len(proof))]) - if _, err := VerifyProof(root, kv.k, proof); err == nil { - t.Fatalf("expected proof to fail for key %x", kv.k) + } +} + +// Tests that missing keys can also be proven. The test explicitly uses a single +// entry trie and checks for missing keys both before and after the single entry. +func TestMissingKeyProof(t *testing.T) { + trie := new(Trie) + updateString(trie, "k", "v") + + for i, key := range []string{"a", "j", "l", "z"} { + proof := trie.Prove([]byte(key)) + if proof == nil { + t.Fatalf("test %d: nil proof", i) + } + if len(proof) != 1 { + t.Errorf("test %d: proof should have one element", i) + } + val, err := VerifyProof(trie.Hash(), []byte(key), proof) + if err != nil { + t.Fatalf("test %d: failed to verify proof: %v\nraw proof: %x", i, err, proof) + } + if val != nil { + t.Fatalf("test %d: verified valuemismatch: have %x, want nil", i, val) } } } @@ -114,7 +162,7 @@ func BenchmarkVerifyProof(b *testing.B) { trie, vals := randomTrie(100) root := trie.Hash() var keys []string - var proofs [][]rlp.RawValue + var proofs [][][]byte for k := range vals { keys = append(keys, k) proofs = append(proofs, trie.Prove([]byte(k))) diff --git a/trie/sync.go b/trie/sync.go index fea10051f4..1130b062db 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -45,11 +45,15 @@ type request struct { callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch } -// SyncResult is a simple list to return missing nodes along with their request -// hashes. +// SyncResult represents a response to a trie node retrieval request. The result +// data might be a simple binary blob if returning only a single node, or it may +// be a batch of trie leaves (with associated merkle proofs) if returning batched +// results. type SyncResult struct { - Hash common.Hash // Hash of the originally unknown trie node - Data []byte // Data content of the retrieved node + Hash common.Hash // Hash of the originally unknown trie node + Data []byte // Data content of the retrieved node, in node-sync mode + Leaves [][]byte // Trie leaves rooted under the specified hash, in leaf-sync mode + Proofs [][]byte // Proofs to validate the leaves, in leaf-sync mode, if leaves are partial } // syncMemBatch is an in-memory buffer of successfully downloaded but not yet diff --git a/trie/sync_test.go b/trie/sync_test.go index ec16a25bd9..7048c08cae 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -120,7 +120,7 @@ func testIterativeTrieSync(t *testing.T, batch int) { if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } - results[i] = SyncResult{hash, data} + results[i] = SyncResult{Hash: hash, Data: data} } if _, index, err := sched.Process(results); err != nil { t.Fatalf("failed to process result #%d: %v", index, err) @@ -153,7 +153,7 @@ func TestIterativeDelayedTrieSync(t *testing.T) { if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } - results[i] = SyncResult{hash, data} + results[i] = SyncResult{Hash: hash, Data: data} } if _, index, err := sched.Process(results); err != nil { t.Fatalf("failed to process result #%d: %v", index, err) @@ -193,7 +193,7 @@ func testIterativeRandomTrieSync(t *testing.T, batch int) { if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, SyncResult{Hash: hash, Data: data}) } // Feed the retrieved results back and queue new tasks if _, index, err := sched.Process(results); err != nil { @@ -233,7 +233,7 @@ func TestIterativeRandomDelayedTrieSync(t *testing.T) { if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, SyncResult{Hash: hash, Data: data}) if len(results) >= cap(results) { break @@ -282,7 +282,7 @@ func TestDuplicateAvoidanceTrieSync(t *testing.T) { } requested[hash] = struct{}{} - results[i] = SyncResult{hash, data} + results[i] = SyncResult{Hash: hash, Data: data} } if _, index, err := sched.Process(results); err != nil { t.Fatalf("failed to process result #%d: %v", index, err) @@ -316,7 +316,7 @@ func TestIncompleteTrieSync(t *testing.T) { if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } - results[i] = SyncResult{hash, data} + results[i] = SyncResult{Hash: hash, Data: data} } // Process each of the trie nodes if _, index, err := sched.Process(results); err != nil {