diff --git a/core/dao_test.go b/core/dao_test.go index e115eddf54..0fbbd7153d 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -82,7 +82,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) @@ -107,7 +107,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) @@ -133,7 +133,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) @@ -153,7 +153,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true); err != nil { + if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 3313103e98..211dcb0a99 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -17,10 +17,10 @@ package state import ( - "bytes" "testing" "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/core/rawdb" ) // Tests that the node iterator indeed walks over the entire database contents. @@ -40,28 +40,54 @@ func TestNodeIteratorCoverage(t *testing.T) { hashes[it.Hash] = struct{}{} } } + // Check in-disk nodes + var ( + seenNodes = make(map[common.Hash]struct{}) + seenCodes = make(map[common.Hash]struct{}) + ) + it := db.NewIterator(nil, nil) + for it.Next() { + ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + if !ok { + continue + } + seenNodes[hash] = struct{}{} + } + it.Release() + + // Check in-disk codes + it = db.NewIterator(nil, nil) + for it.Next() { + ok, hash := rawdb.IsCodeKey(it.Key()) + if !ok { + continue + } + if _, ok := hashes[common.BytesToHash(hash)]; !ok { + t.Errorf("state entry not reported %x", it.Key()) + } + seenCodes[common.BytesToHash(hash)] = struct{}{} + } + it.Release() + // Cross check the iterated hashes and the database/nodepool content for hash := range hashes { - if _, err = sdb.TrieDB().Node(hash); err != nil { - _, err = sdb.ContractCode(common.Hash{}, hash) + _, ok := seenNodes[hash] + if !ok { + _, ok = seenCodes[hash] } - if err != nil { + if !ok { t.Errorf("failed to retrieve reported node %x", hash) } } - for _, hash := range sdb.TrieDB().Nodes() { - if _, ok := hashes[hash]; !ok { - t.Errorf("state entry not reported %x", hash) - } - } - it := db.NewIterator(nil, nil) - for it.Next() { - key := it.Key() - if bytes.HasPrefix(key, []byte("secure-key-")) { - continue - } - if _, ok := hashes[common.BytesToHash(key)]; !ok { - t.Errorf("state entry not reported %x", key) - } - } +} + +// isTrieNode is a helper function which reports if the provided +// database entry belongs to a trie node or not. +func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { + if scheme == rawdb.HashScheme { + if len(key) == common.HashLength { + return true, common.BytesToHash(key) + } + } + return false, common.Hash{} } diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 65e775c325..d16d25c359 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -79,17 +79,17 @@ func TestHexKeybytes(t *testing.T) { } func TestHexToCompactInPlace(t *testing.T) { - for i, keyS := range []string{ + for i, key := range []string{ "00", "060a040c0f000a090b040803010801010900080d090a0a0d0903000b10", "10", } { - hexBytes, _ := hex.DecodeString(keyS) + hexBytes, _ := hex.DecodeString(key) exp := hexToCompact(hexBytes) sz := hexToCompactInPlace(hexBytes) got := hexBytes[:sz] if !bytes.Equal(exp, got) { - t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, keyS, got, exp) + t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, key, got, exp) } } } diff --git a/trie/iterator.go b/trie/iterator.go index a42ce9f36d..eed9098657 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -23,9 +23,15 @@ import ( "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core/types" - "github.com/XinFinOrg/XDPoSChain/ethdb" ) +// NodeResolver is used for looking up trie nodes before reaching into the real +// persistent layer. This is not mandatory, rather is an optimization for cases +// where trie nodes can be recovered from some external mechanism without reading +// from disk. In those cases, this resolver allows short circuiting accesses and +// returning them from memory. +type NodeResolver func(owner common.Hash, path []byte, hash common.Hash) []byte + // Iterator is a key-value trie iterator that traverses a Trie. type Iterator struct { nodeIt NodeIterator @@ -108,8 +114,8 @@ type NodeIterator interface { // to the value after calling Next. LeafProof() [][]byte - // AddResolver sets an intermediate database to use for looking up trie nodes - // before reaching into the real persistent layer. + // AddResolver sets a node resolver to use for looking up trie nodes before + // reaching into the real persistent layer. // // This is not required for normal operation, rather is an optimization for // cases where trie nodes can be recovered from some external mechanism without @@ -119,7 +125,7 @@ type NodeIterator interface { // Before adding a similar mechanism to any other place in Geth, consider // making trie.Database an interface and wrapping at that level. It's a huge // refactor, but it could be worth it if another occurrence arises. - AddResolver(ethdb.KeyValueReader) + AddResolver(NodeResolver) } // nodeIteratorState represents the iteration state at one particular Node of the @@ -138,7 +144,7 @@ type nodeIterator struct { path []byte // Path to the current Node err error // Failure set in case of an internal error in the iterator - resolver ethdb.KeyValueReader // Optional intermediate resolver above the disk layer + resolver NodeResolver // optional node resolver for avoiding disk hits } // errIteratorEnd is stored in nodeIterator.err when iteration is done. @@ -166,7 +172,7 @@ func newNodeIterator(trie *Trie, start []byte) NodeIterator { return it } -func (it *nodeIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *nodeIterator) AddResolver(resolver NodeResolver) { it.resolver = resolver } @@ -370,7 +376,7 @@ func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []by func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { if it.resolver != nil { - if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + if blob := it.resolver(it.trie.owner, path, common.BytesToHash(hash)); len(blob) > 0 { if resolved, err := decodeNode(hash, blob); err == nil { return resolved, nil } @@ -386,7 +392,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { if it.resolver != nil { - if blob, err := it.resolver.Get(hash); err == nil && len(blob) > 0 { + if blob := it.resolver(it.trie.owner, path, common.BytesToHash(hash)); len(blob) > 0 { return blob, nil } } @@ -590,7 +596,7 @@ func (it *differenceIterator) NodeBlob() []byte { return it.b.NodeBlob() } -func (it *differenceIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *differenceIterator) AddResolver(resolver NodeResolver) { panic("not implemented") } @@ -705,7 +711,7 @@ func (it *unionIterator) NodeBlob() []byte { return (*it.items)[0].NodeBlob() } -func (it *unionIterator) AddResolver(resolver ethdb.KeyValueReader) { +func (it *unionIterator) AddResolver(resolver NodeResolver) { panic("not implemented") } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 4c329d9a1b..f1f21cd2d6 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -337,7 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { _, nodes, _ := tr.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(tr.Hash(), true) + triedb.Commit(tr.Hash(), false) } wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil) @@ -429,7 +429,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { root, nodes, _ := ctr.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, true) + triedb.Commit(root, false) } barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e") var ( diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 3250c497af..f5e748f564 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -255,7 +255,6 @@ func TestValLength56(t *testing.T) { func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - kvs := []struct { K string V string @@ -284,7 +283,6 @@ func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - kvs := []struct { K string V string diff --git a/trie/trie_test.go b/trie/trie_test.go index c679e7f8c9..70790a63de 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -88,7 +88,7 @@ func testMissingNode(t *testing.T, memonly bool) { root, nodes, _ := trie.Commit(false) triedb.Update(NewWithNodeSet(nodes)) if !memonly { - triedb.Commit(root, true) + triedb.Commit(root, false) } trie, _ = New(TrieID(root), triedb) @@ -809,6 +809,80 @@ func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } +// TestCommitSequence tests that the trie.Commit operation writes the elements of the trie +// in the expected order. +// The test data was based on the 'master' code, and is basically random. It can be used +// to check whether changes to the trie modifies the write order or data in any way. +func TestCommitSequence(t *testing.T) { + for i, tc := range []struct { + count int + expWriteSeqHash []byte + }{ + {20, common.FromHex("873c78df73d60e59d4a2bcf3716e8bfe14554549fea2fc147cb54129382a8066")}, + {200, common.FromHex("ba03d891bb15408c940eea5ee3d54d419595102648d02774a0268d892add9c8e")}, + {2000, common.FromHex("f7a184f20df01c94f09537401d11e68d97ad0c00115233107f51b9c287ce60c7")}, + } { + addresses, accounts := makeAccounts(tc.count) + // This spongeDb is used to check the sequence of disk-db-writes + s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} + db := NewDatabase(rawdb.NewDatabase(s)) + trie := NewEmpty(db) + // Fill the trie with elements + for i := 0; i < tc.count; i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Flush trie -> database + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + // Flush memdb -> disk (sponge) + db.Commit(root, false) + if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { + t.Errorf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) + } + } +} + +// TestCommitSequenceRandomBlobs is identical to TestCommitSequence +// but uses random blobs instead of 'accounts' +func TestCommitSequenceRandomBlobs(t *testing.T) { + for i, tc := range []struct { + count int + expWriteSeqHash []byte + }{ + {20, common.FromHex("8e4a01548551d139fa9e833ebc4e66fc1ba40a4b9b7259d80db32cff7b64ebbc")}, + {200, common.FromHex("6869b4e7b95f3097a19ddb30ff735f922b915314047e041614df06958fc50554")}, + {2000, common.FromHex("444200e6f4e2df49f77752f629a96ccf7445d4698c164f962bbd85a0526ef424")}, + } { + prng := rand.New(rand.NewSource(int64(i))) + // This spongeDb is used to check the sequence of disk-db-writes + s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} + db := NewDatabase(rawdb.NewDatabase(s)) + trie := NewEmpty(db) + // Fill the trie with elements + for i := 0; i < tc.count; i++ { + key := make([]byte, 32) + var val []byte + // 50% short elements, 50% large elements + if prng.Intn(2) == 0 { + val = make([]byte, 1+prng.Intn(32)) + } else { + val = make([]byte, 1+prng.Intn(4096)) + } + prng.Read(key) + prng.Read(val) + trie.Update(key, val) + } + // Flush trie -> database + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + // Flush memdb -> disk (sponge) + db.Commit(root, false) + if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { + t.Fatalf("test %d, disk write sequence wrong:\ngot %x exp %x\n", i, got, exp) + } + } +} + func TestCommitSequenceStackTrie(t *testing.T) { for count := 1; count < 200; count++ { prng := rand.New(rand.NewSource(int64(count)))