trie: upgarde for snap protocol #21482 (#1058)

trie/notary.go

@@ -0,0 +1,57 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"github.com/XinFinOrg/XDPoSChain/ethdb"
+	"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
+)
+
+// KeyValueNotary tracks which keys have been accessed through a key-value reader
+// with te scope of verifying if certain proof datasets are maliciously bloated.
+type KeyValueNotary struct {
+	ethdb.KeyValueReader
+	reads map[string]struct{}
+}
+
+// NewKeyValueNotary wraps a key-value database with an access notary to track
+// which items have bene accessed.
+func NewKeyValueNotary(db ethdb.KeyValueReader) *KeyValueNotary {
+	return &KeyValueNotary{
+		KeyValueReader: db,
+		reads:          make(map[string]struct{}),
+	}
+}
+
+// Get retrieves an item from the underlying database, but also tracks it as an
+// accessed slot for bloat checks.
+func (k *KeyValueNotary) Get(key []byte) ([]byte, error) {
+	k.reads[string(key)] = struct{}{}
+	return k.KeyValueReader.Get(key)
+}
+
+// Accessed returns s snapshot of the original key-value store containing only the
+// data accessed through the notary.
+func (k *KeyValueNotary) Accessed() ethdb.KeyValueStore {
+	db := memorydb.New()
+	for keystr := range k.reads {
+		key := []byte(keystr)
+		val, _ := k.KeyValueReader.Get(key)
+		db.Put(key, val)
+	}
+	return db
+}

trie/proof.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
-	"github.com/XinFinOrg/XDPoSChain/core/types"
 	"github.com/XinFinOrg/XDPoSChain/ethdb"
 	"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
 	"github.com/XinFinOrg/XDPoSChain/log"
@@ -426,7 +425,7 @@ func hasRightElement(node node, key []byte) bool {
 
 // VerifyRangeProof checks whether the given leaf nodes and edge proof
 // can prove the given trie leaves range is matched with the specific root.
-// Besides, the range should be consecutive(no gap inside) and monotonic
+// Besides, the range should be consecutive (no gap inside) and monotonic
 // increasing.
 //
 // Note the given proof actually contains two edge proofs. Both of them can
@@ -454,96 +453,136 @@ func hasRightElement(node node, key []byte) bool {
 //
 // Except returning the error to indicate the proof is valid or not, the function will
 // also return a flag to indicate whether there exists more accounts/slots in the trie.
-func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (error, bool) {
+func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (ethdb.KeyValueStore, *Trie, *KeyValueNotary, bool, error) {
 	if len(keys) != len(values) {
-		return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false
+		return nil, nil, nil, false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values))
 	}
 	// Ensure the received batch is monotonic increasing.
 	for i := 0; i < len(keys)-1; i++ {
 		if bytes.Compare(keys[i], keys[i+1]) >= 0 {
-			return errors.New("range is not monotonically increasing"), false
+			return nil, nil, nil, false, errors.New("range is not monotonically increasing")
 		}
 	}
+	// Create a key-value notary to track which items from the given proof the
+	// range prover actually needed to verify the data
+	notary := NewKeyValueNotary(proof)
+
 	// Special case, there is no edge proof at all. The given range is expected
 	// to be the whole leaf-set in the trie.
 	if proof == nil {
-		emptytrie, err := New(types.EmptyRootHash, NewDatabase(memorydb.New()))
+		var (
+			diskdb = memorydb.New()
+			triedb = NewDatabase(diskdb)
+		)
+		tr, err := New(common.Hash{}, triedb)
 		if err != nil {
-			return err, false
+			return nil, nil, nil, false, err
 		}
 		for index, key := range keys {
-			emptytrie.TryUpdate(key, values[index])
+			tr.TryUpdate(key, values[index])
 		}
-		if emptytrie.Hash() != rootHash {
+		if tr.Hash() != rootHash {
-			return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, emptytrie.Hash()), false
+			return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash())
 		}
-		return nil, false // no more element.
+		// Proof seems valid, serialize all the nodes into the database
+		if _, err := tr.Commit(nil); err != nil {
+			return nil, nil, nil, false, err
+		}
+		if err := triedb.Commit(rootHash, false); err != nil {
+			return nil, nil, nil, false, err
+		}
+		return diskdb, tr, notary, false, nil // No more elements
 	}
 	// Special case, there is a provided edge proof but zero key/value
 	// pairs, ensure there are no more accounts / slots in the trie.
 	if len(keys) == 0 {
-		root, val, err := proofToPath(rootHash, nil, firstKey, proof, true)
+		root, val, err := proofToPath(rootHash, nil, firstKey, notary, true)
 		if err != nil {
-			return err, false
+			return nil, nil, nil, false, err
 		}
 		if val != nil || hasRightElement(root, firstKey) {
-			return errors.New("more entries available"), false
+			return nil, nil, nil, false, errors.New("more entries available")
 		}
-		return nil, false
+		// Since the entire proof is a single path, we can construct a trie and a
+		// node database directly out of the inputs, no need to generate them
+		diskdb := notary.Accessed()
+		tr := &Trie{
+			Db:   NewDatabase(diskdb),
+			root: root,
+		}
+		return diskdb, tr, notary, hasRightElement(root, firstKey), nil
 	}
 	// Special case, there is only one element and two edge keys are same.
 	// In this case, we can't construct two edge paths. So handle it here.
 	if len(keys) == 1 && bytes.Equal(firstKey, lastKey) {
-		root, val, err := proofToPath(rootHash, nil, firstKey, proof, false)
+		root, val, err := proofToPath(rootHash, nil, firstKey, notary, false)
 		if err != nil {
-			return err, false
+			return nil, nil, nil, false, err
 		}
 		if !bytes.Equal(firstKey, keys[0]) {
-			return errors.New("correct proof but invalid key"), false
+			return nil, nil, nil, false, errors.New("correct proof but invalid key")
 		}
 		if !bytes.Equal(val, values[0]) {
-			return errors.New("correct proof but invalid data"), false
+			return nil, nil, nil, false, errors.New("correct proof but invalid data")
 		}
-		return nil, hasRightElement(root, firstKey)
+		// Since the entire proof is a single path, we can construct a trie and a
+		// node database directly out of the inputs, no need to generate them
+		diskdb := notary.Accessed()
+		tr := &Trie{
+			Db:   NewDatabase(diskdb),
+			root: root,
+		}
+		return diskdb, tr, notary, hasRightElement(root, firstKey), nil
 	}
 	// Ok, in all other cases, we require two edge paths available.
 	// First check the validity of edge keys.
 	if bytes.Compare(firstKey, lastKey) >= 0 {
-		return errors.New("invalid edge keys"), false
+		return nil, nil, nil, false, errors.New("invalid edge keys")
 	}
 	// todo(rjl493456442) different length edge keys should be supported
 	if len(firstKey) != len(lastKey) {
-		return errors.New("inconsistent edge keys"), false
+		return nil, nil, nil, false, errors.New("inconsistent edge keys")
 	}
 	// Convert the edge proofs to edge trie paths. Then we can
 	// have the same tree architecture with the original one.
 	// For the first edge proof, non-existent proof is allowed.
-	root, _, err := proofToPath(rootHash, nil, firstKey, proof, true)
+	root, _, err := proofToPath(rootHash, nil, firstKey, notary, true)
 	if err != nil {
-		return err, false
+		return nil, nil, nil, false, err
 	}
 	// Pass the root node here, the second path will be merged
 	// with the first one. For the last edge proof, non-existent
 	// proof is also allowed.
-	root, _, err = proofToPath(rootHash, root, lastKey, proof, true)
+	root, _, err = proofToPath(rootHash, root, lastKey, notary, true)
 	if err != nil {
-		return err, false
+		return nil, nil, nil, false, err
 	}
 	// Remove all internal references. All the removed parts should
 	// be re-filled(or re-constructed) by the given leaves range.
 	if err := unsetInternal(root, firstKey, lastKey); err != nil {
-		return err, false
+		return nil, nil, nil, false, err
 	}
-	// Rebuild the trie with the leave stream, the shape of trie
+	// Rebuild the trie with the leaf stream, the shape of trie
 	// should be same with the original one.
-	newtrie := &Trie{root: root, Db: NewDatabase(memorydb.New())}
+	var (
+		diskdb = memorydb.New()
+		triedb = NewDatabase(diskdb)
+	)
+	tr := &Trie{root: root, Db: triedb}
 	for index, key := range keys {
-		newtrie.TryUpdate(key, values[index])
+		tr.TryUpdate(key, values[index])
 	}
-	if newtrie.Hash() != rootHash {
+	if tr.Hash() != rootHash {
-		return fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, newtrie.Hash()), false
+		return nil, nil, nil, false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash())
 	}
-	return nil, hasRightElement(root, keys[len(keys)-1])
+	// Proof seems valid, serialize all the nodes into the database
+	if _, err := tr.Commit(nil); err != nil {
+		return nil, nil, nil, false, err
+	}
+	if err := triedb.Commit(rootHash, false); err != nil {
+		return nil, nil, nil, false, err
+	}
+	return diskdb, tr, notary, hasRightElement(root, keys[len(keys)-1]), nil
 }
 
 // get returns the child of the given Node. Return nil if the

trie/proof_test.go

@@ -19,6 +19,7 @@ package trie
 import (
 	"bytes"
 	crand "crypto/rand"
+	"encoding/binary"
 	mrand "math/rand"
 	"sort"
 	"testing"
@@ -176,7 +177,7 @@ func TestRangeProof(t *testing.T) {
 			keys = append(keys, entries[i].k)
 			vals = append(vals, entries[i].v)
 		}
-		err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
+		_, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
 		if err != nil {
 			t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
 		}
@@ -227,7 +228,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) {
 			keys = append(keys, entries[i].k)
 			vals = append(vals, entries[i].v)
 		}
-		err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
+		_, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
 		if err != nil {
 			t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
 		}
@@ -248,7 +249,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) {
 		k = append(k, entries[i].k)
 		v = append(v, entries[i].v)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
 	if err != nil {
 		t.Fatal("Failed to verify whole rang with non-existent edges")
 	}
@@ -283,7 +284,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) {
 		k = append(k, entries[i].k)
 		v = append(v, entries[i].v)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof)
 	if err == nil {
 		t.Fatalf("Expected to detect the error, got nil")
 	}
@@ -305,7 +306,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) {
 		k = append(k, entries[i].k)
 		v = append(v, entries[i].v)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof)
 	if err == nil {
 		t.Fatalf("Expected to detect the error, got nil")
 	}
@@ -329,7 +330,7 @@ func TestOneElementRangeProof(t *testing.T) {
 	if err := trie.Prove(entries[start].k, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the first node %v", err)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -344,7 +345,7 @@ func TestOneElementRangeProof(t *testing.T) {
 	if err := trie.Prove(entries[start].k, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -359,7 +360,7 @@ func TestOneElementRangeProof(t *testing.T) {
 	if err := trie.Prove(last, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -374,7 +375,7 @@ func TestOneElementRangeProof(t *testing.T) {
 	if err := trie.Prove(last, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -396,7 +397,7 @@ func TestAllElementsProof(t *testing.T) {
 		k = append(k, entries[i].k)
 		v = append(v, entries[i].v)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -409,7 +410,7 @@ func TestAllElementsProof(t *testing.T) {
 	if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -424,7 +425,7 @@ func TestAllElementsProof(t *testing.T) {
 	if err := trie.Prove(last, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
 	if err != nil {
 		t.Fatalf("Expected no error, got %v", err)
 	}
@@ -457,7 +458,7 @@ func TestSingleSideRangeProof(t *testing.T) {
 				k = append(k, entries[i].k)
 				v = append(v, entries[i].v)
 			}
-			err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof)
+			_, _, _, _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof)
 			if err != nil {
 				t.Fatalf("Expected no error, got %v", err)
 			}
@@ -493,7 +494,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) {
 				k = append(k, entries[i].k)
 				v = append(v, entries[i].v)
 			}
-			err, _ := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof)
+			_, _, _, _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof)
 			if err != nil {
 				t.Fatalf("Expected no error, got %v", err)
 			}
@@ -565,7 +566,7 @@ func TestBadRangeProof(t *testing.T) {
 			index = mrand.Intn(end - start)
 			vals[index] = nil
 		}
-		err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
+		_, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
 		if err == nil {
 			t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1)
 		}
@@ -599,7 +600,7 @@ func TestGappedRangeProof(t *testing.T) {
 		keys = append(keys, entries[i].k)
 		vals = append(vals, entries[i].v)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
 	if err == nil {
 		t.Fatal("expect error, got nil")
 	}
@@ -626,7 +627,7 @@ func TestSameSideProofs(t *testing.T) {
 	if err := trie.Prove(last, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
+	_, _, _, _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
 	if err == nil {
 		t.Fatalf("Expected error, got nil")
 	}
@@ -642,7 +643,7 @@ func TestSameSideProofs(t *testing.T) {
 	if err := trie.Prove(last, 0, proof); err != nil {
 		t.Fatalf("Failed to prove the last node %v", err)
 	}
-	err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
+	_, _, _, _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
 	if err == nil {
 		t.Fatalf("Expected error, got nil")
 	}
@@ -710,7 +711,7 @@ func TestHasRightElement(t *testing.T) {
 			k = append(k, entries[i].k)
 			v = append(v, entries[i].v)
 		}
-		err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof)
+		_, _, _, hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof)
 		if err != nil {
 			t.Fatalf("Expected no error, got %v", err)
 		}
@@ -743,13 +744,57 @@ func TestEmptyRangeProof(t *testing.T) {
 		if err := trie.Prove(first, 0, proof); err != nil {
 			t.Fatalf("Failed to prove the first node %v", err)
 		}
-		err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof)
+		db, tr, not, _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof)
 		if c.err && err == nil {
 			t.Fatalf("Expected error, got nil")
 		}
 		if !c.err && err != nil {
 			t.Fatalf("Expected no error, got %v", err)
 		}
+		// If no error was returned, ensure the returned trie and database contains
+		// the entire proof, since there's no value
+		if !c.err {
+			if err := tr.Prove(first, 0, memorydb.New()); err != nil {
+				t.Errorf("returned trie doesn't contain original proof: %v", err)
+			}
+			if memdb := db.(*memorydb.Database); memdb.Len() != proof.Len() {
+				t.Errorf("database entry count mismatch: have %d, want %d", memdb.Len(), proof.Len())
+			}
+			if not == nil {
+				t.Errorf("missing notary")
+			}
+		}
+	}
+}
+
+// TestBloatedProof tests a malicious proof, where the proof is more or less the
+// whole trie.
+func TestBloatedProof(t *testing.T) {
+	// Use a small trie
+	trie, kvs := nonRandomTrie(100)
+	var entries entrySlice
+	for _, kv := range kvs {
+		entries = append(entries, kv)
+	}
+	sort.Sort(entries)
+	var keys [][]byte
+	var vals [][]byte
+
+	proof := memorydb.New()
+	for i, entry := range entries {
+		trie.Prove(entry.k, 0, proof)
+		if i == 50 {
+			keys = append(keys, entry.k)
+			vals = append(vals, entry.v)
+		}
+	}
+	want := memorydb.New()
+	trie.Prove(keys[0], 0, want)
+	trie.Prove(keys[len(keys)-1], 0, want)
+
+	_, _, notary, _, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
+	if used := notary.Accessed().(*memorydb.Database); used.Len() != want.Len() {
+		t.Fatalf("notary proof size mismatch: have %d, want %d", used.Len(), want.Len())
 	}
 }
 
@@ -853,7 +898,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof)
+		_, _, _, _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof)
 		if err != nil {
 			b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
 		}
@@ -884,3 +929,20 @@ func randBytes(n int) []byte {
 	crand.Read(r)
 	return r
 }
+
+func nonRandomTrie(n int) (*Trie, map[string]*kv) {
+	trie := new(Trie)
+	vals := make(map[string]*kv)
+	max := uint64(0xffffffffffffffff)
+	for i := uint64(0); i < uint64(n); i++ {
+		value := make([]byte, 32)
+		key := make([]byte, 32)
+		binary.LittleEndian.PutUint64(key, i)
+		binary.LittleEndian.PutUint64(value, i-max)
+		//value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
+		elem := &kv{key, value, false}
+		trie.Update(elem.k, elem.v)
+		vals[string(elem.k)] = elem
+	}
+	return trie, vals
+}

trie/trie.go

@@ -19,13 +19,13 @@ package trie
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"sync"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/core/types"
 	"github.com/XinFinOrg/XDPoSChain/log"
-	"github.com/XinFinOrg/XDPoSChain/rlp"
 )
 
 // LeafCallback is a callback type invoked when a trie operation reaches a leaf
@@ -162,29 +162,26 @@ func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) {
 	if item == nil {
 		return nil, resolved, nil
 	}
-	enc, err := rlp.EncodeToBytes(item)
+	return item, resolved, err
-	if err != nil {
-		log.Error("Encoding existing trie node failed", "err", err)
-		return nil, resolved, err
-	}
-	return enc, resolved, err
 }
 
-func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item node, newnode node, resolved int, err error) {
+func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) {
 	// If we reached the requested path, return the current node
 	if pos >= len(path) {
-		// Don't return collapsed hash nodes though
+		// Although we most probably have the original node expanded, encoding
-		if _, ok := origNode.(hashNode); !ok {
+		// that into consensus form can be nasty (needs to cascade down) and
-			// Short nodes have expanded keys, compact them before returning
+		// time consuming. Instead, just pull the hash up from disk directly.
-			item := origNode
+		var hash hashNode
-			if sn, ok := item.(*shortNode); ok {
+		if node, ok := origNode.(hashNode); ok {
-				item = &shortNode{
+			hash = node
-					Key: hexToCompact(sn.Key),
+		} else {
-					Val: sn.Val,
+			hash, _ = origNode.cache()
-				}
-			}
-			return item, origNode, 0, nil
 		}
+		if hash == nil {
+			return nil, origNode, 0, errors.New("non-consensus node")
+		}
+		blob, err := t.Db.Node(common.BytesToHash(hash))
+		return blob, origNode, 1, err
 	}
 	// Path still needs to be traversed, descend into children
 	switch n := (origNode).(type) {